queue_classic_plus 0.0.2 → 1.0.0.alpha2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8acdaf6cd41362f50d7ddb9c50fa87555c00adc9
-  data.tar.gz: 58c26d31a05c2390504975b6a7513179f0397ede
+  metadata.gz: f5b1934a03303e576a999e71f978a68da13f0298
+  data.tar.gz: d1de5250aa7b2ddfe4114043d87e5a95986145f8
 SHA512:
-  metadata.gz: 453a051c64c4c539f488f9ea7f0c01da9ac326de11891746b6b0abea14bc4103bb347afd4020bedb0b37c0262d1c1a983659f39c0b35909521db6af2951dcacb
-  data.tar.gz: 37263c41d6929a351e74920a8abe5d2d4d3bbc3c5c9bdb941f4cd2e6758ef2443ddf580dd16298124809ca0232ce7df02b3533fac75fef8fa911f21c45ecb597
+  metadata.gz: ebadc9325c0307789f6a6216047a2c2cfe240d4364b613ee9eefc9a479224a9196813146b90ed251124cd5ad7a4966556c8fca8e1b48a490cfd806bf95124ff3
+  data.tar.gz: 619c736ed7b74f94841328acfa27926e2799fecc6589974d9b79de236c1f396ef126d1e80205df223e626536b69c1c0a834014e09e190b000635e571c2a8e677

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+before_script:
+  - psql -c 'create database queue_classic_plus_test;' -U postgres
+
+rvm:
+  - 2.0.0
+  - 2.1.5
+  - 2.2.0

data/Gemfile

@@ -3,9 +3,15 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in queue_classic_plus.gemspec
 gemspec
 
+gem "queue_classic_matchers", github: 'rainforestapp/queue_classic_matchers'
+gem 'pry'
 
-gem 'rspec'
-gem 'pg'
-gem 'timecop'
-gem 'queue_classic-later', github: 'jipiboily/queue_classic-later', branch: "add-qc-3-to-custom-columns" # This is until the 3.0 work is merged into original repo
-gem 'queue_classic_matchers'
+group :development do
+  gem "guard-rspec", require: false
+  gem "terminal-notifier-guard"
+end
+
+group :test do
+  gem 'rspec'
+  gem 'timecop'
+end

data/Guardfile

@@ -0,0 +1,77 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+## Uncomment and set this to only include directories you want to watch
+# directories %w(app lib config test spec feature)
+
+## Uncomment to clear the screen before every task
+# clearing :on
+
+## Guard internally checks for changes in the Guardfile and exits.
+## If you want Guard to automatically start up again, run guard in a
+## shell loop, e.g.:
+##
+##  $ while bundle exec guard; do echo "Restarting Guard..."; done
+##
+## Note: if you are using the `directories` clause above and you are not
+## watching the project directory ('.'), the you will want to move the Guardfile
+## to a watched dir and symlink it back, e.g.
+#
+#  $ mkdir config
+#  $ mv Guardfile config/
+#  $ ln -s config/Guardfile .
+#
+# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
+
+# Note: The cmd option is now required due to the increasing number of ways
+#       rspec may be run, below are examples of the most common uses.
+#  * bundler: 'bundle exec rspec'
+#  * bundler binstubs: 'bin/rspec'
+#  * spring: 'bin/rspec' (This will use spring if running and you have
+#                          installed the spring binstubs per the docs)
+#  * zeus: 'zeus rspec' (requires the server to be started separately)
+#  * 'just' rspec: 'rspec'
+
+guard :rspec, cmd: "bundle exec rspec" do
+  require "guard/rspec/dsl"
+  dsl = Guard::RSpec::Dsl.new(self)
+
+  # Feel free to open issues for suggestions and improvements
+
+  # RSpec files
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+
+  # Rails files
+  rails = dsl.rails(view_extensions: %w(erb haml slim))
+  dsl.watch_spec_files_for(rails.app_files)
+  dsl.watch_spec_files_for(rails.views)
+
+  watch(rails.controllers) do |m|
+    [
+      rspec.spec.("routing/#{m[1]}_routing"),
+      rspec.spec.("controllers/#{m[1]}_controller"),
+      rspec.spec.("acceptance/#{m[1]}")
+    ]
+  end
+
+  # Rails config changes
+  watch(rails.spec_helper)     { rspec.spec_dir }
+  watch(rails.routes)          { "#{rspec.spec_dir}/routing" }
+  watch(rails.app_controller)  { "#{rspec.spec_dir}/controllers" }
+
+  # Capybara features specs
+  watch(rails.view_dirs)     { |m| rspec.spec.("features/#{m[1]}") }
+
+  # Turnip features and steps
+  watch(%r{^spec/acceptance/(.+)\.feature$})
+  watch(%r{^spec/acceptance/steps/(.+)_steps\.rb$}) do |m|
+    Dir[File.join("**/#{m[1]}.feature")][0] || "spec/acceptance"
+  end
+end

data/README.md

@@ -1,6 +1,8 @@
 # QueueClassicPlus
 
-QueueClassic is a simple Postgresql back DB queue. However, it's a little too simple to use it as the main queueing system of a medium to large app.
+[![Build Status](https://travis-ci.org/rainforestapp/queue_classic_plus.svg?branch=master)](https://travis-ci.org/rainforestapp/queue_classic_plus)
+
+[QueueClassic](https://github.com/QueueClassic/queue_classic) is a simple Postgresql back DB queue. However, it's a little too simple to use it as the main queueing system of a medium to large app.
 
 QueueClassicPlus adds many lacking features to QueueClassic.
 
@@ -10,13 +12,17 @@ QueueClassicPlus adds many lacking features to QueueClassic.
 - Metrics
 - Error logging / handling
 - Transactions
+- Rails generator to create new jobs
+
+## Compatibility
+
+This version of the matchers are compatible with queue_classic 3.1+ which includes built-in scheduling. See other branches for other compatible versions.
 
 ## Installation
 
 Add these line to your application's Gemfile:
 
     gem 'queue_classic_plus'
-    gem "queue_classic-later", github: "dpiddy/queue_classic-later"
 
 And then execute:
 
@@ -33,7 +39,7 @@ Run the migration
 ### Create a new job
 
 ```bash
-rails g my_job test_job
+rails g qc_plus_job test_job
 ```
 
 ```ruby
@@ -54,17 +60,67 @@ end
 Jobs::MyJob.do(1, "foo")
 
 # You can also schedule a job in the future by doing
-
 Jobs::MyJob.enqueue_perform_in(1.hour, 1, "foo")
 ```
 
 ### Run the QueueClassicPlus worker
 
+QueueClassicPlus ships with its own worker and a rake task to run it. You need to use this worker to take advance of many features of QueueClassicPlus.
+
 ```
 QUEUE=low bundle exec qc_plus:work
 ```
 
-## Advance configuration
+### Other jobs options
+
+#### Singleton Job
+
+It's common for background jobs to never need to be enqueed multiple time. QueueClassicPlus support these type of single jobs. Here's an example one:
+
+```ruby
+class Jobs::UpdateMetrics < QueueClassicPlus::Base
+  @queue = :low
+
+  # Use the lock! keyword to prevent the job from being enqueud once.
+  lock!
+
+  def self.perform(metric_type)
+    # ...
+  end
+end
+
+```
+
+Note that `lock!` only prevents the same job from beeing enqued multiple times if the argument match.
+
+So in our example:
+
+```ruby
+Jobs::UpdateMetrics.do 'type_a' # enqueues job
+Jobs::UpdateMetrics.do 'type_a' # does not enqueues job since it's already queued
+Jobs::UpdateMetrics.do 'type_b' # enqueues job as the arguments are different.
+```
+
+```ruby
+class Jobs::NoTransaction < QueueClassicPlus::Base
+  # Don't run the perform method in a transaction
+  skip_transaction!
+
+  @queue = :low
+
+  def self.perform(user_id)
+    # ...
+  end
+end
+```
+
+#### Transaction
+
+By default, all QueueClassicPlus jobs are executed in a PostgreSQL transaction. This decision was made because most jobs are usually pretty small and it's preferable to have all the benefits of the transaction.
+
+You can disable this feature on a per job basis in the follwing way:
+
+## Advanced configuration
 
 If you want to log exceptions in your favorite exception tracker. You can configured it like sso:
 
@@ -84,14 +140,22 @@ QueueClassicPlus.update_metrics
 
 Call this is a cron job or something similar.
 
-## TODO
+If you are using NewRelic and want to push performance data to it, you can add this to an initializer:
 
-- Remove dep on ActiveRecord
+```ruby
+require "queue_classic_plus/new_relic"
+```
 
 ## Contributing
 
 1. Fork it ( https://github.com/[my-github-username]/queue_classic_plus/fork )
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create a new Pull Request
+- Create your feature branch (`git checkout -b my-new-feature`)
+- Commit your changes (`git commit -am 'Add some feature'`)
+- Push to the branch (`git push origin my-new-feature`)
+- Create a new Pull Request
+
+### Setting up the test database
+
+```
+createdb queue_classic_plus_test
+```

data/Rakefile

@@ -1,2 +1,11 @@
 require "bundler/gem_tasks"
 
+begin
+  require 'rspec/core/rake_task'
+
+  RSpec::Core::RakeTask.new(:spec)
+
+  task :default => :spec
+rescue LoadError
+  # no rspec available
+end

data/lib/queue_classic_plus.rb

@@ -1,14 +1,14 @@
+require 'logger'
 require "queue_classic"
-require "queue_classic/later"
-require "active_record"
-require "with_advisory_lock"
-require "active_support/core_ext/class/attribute"
 
 require "queue_classic_plus/version"
+require "queue_classic_plus/inheritable_attr"
+require "queue_classic_plus/inflector"
 require "queue_classic_plus/metrics"
 require "queue_classic_plus/update_metrics"
 require "queue_classic_plus/base"
 require "queue_classic_plus/worker"
+require "queue_classic_plus/queue_classic/queue"
 
 module QueueClassicPlus
   require 'queue_classic_plus/railtie' if defined?(Rails)
@@ -17,14 +17,12 @@ module QueueClassicPlus
     conn = QC::ConnAdapter.new(c)
     conn.execute("ALTER TABLE queue_classic_jobs ADD COLUMN last_error TEXT")
     conn.execute("ALTER TABLE queue_classic_jobs ADD COLUMN remaining_retries INTEGER")
-    conn.execute("ALTER TABLE queue_classic_later_jobs ADD COLUMN remaining_retries INTEGER")
   end
 
   def self.demigrate(c = QC::default_conn_adapter.connection)
     conn = QC::ConnAdapter.new(c)
     conn.execute("ALTER TABLE queue_classic_jobs DROP COLUMN last_error")
     conn.execute("ALTER TABLE queue_classic_jobs DROP COLUMN remaining_retries")
-    conn.execute("ALTER TABLE queue_classic_later_jobs DROP COLUMN remaining_retries")
   end
 
   def self.exception_handler

data/lib/queue_classic_plus/base.rb

@@ -1,13 +1,15 @@
 module QueueClassicPlus
   class Base
+    extend QueueClassicPlus::InheritableAttribute
+
     def self.queue
       QC::Queue.new(@queue)
     end
 
-    class_attribute :locked
-    class_attribute :skip_transaction
-    class_attribute :retries_on
-    class_attribute :max_retries
+    inheritable_attr :locked
+    inheritable_attr :skip_transaction
+    inheritable_attr :retries_on
+    inheritable_attr :max_retries
 
     self.max_retries = 0
     self.retries_on = {}
@@ -39,35 +41,21 @@ module QueueClassicPlus
 
     def self.can_enqueue?(method, *args)
       if locked?
-        lock_key = [@queue, method, args].to_json
         max_lock_time = ENV.fetch("QUEUE_CLASSIC_MAX_LOCK_TIME", 10 * 60).to_i
 
-        ActiveRecord::Base.with_advisory_lock(lock_key, 0.1) do
-          q = "SELECT COUNT(1) AS count
-               FROM
+        q = "SELECT COUNT(1) AS count
+             FROM
                (
-                 (
                  SELECT 1
                  FROM queue_classic_jobs
                  WHERE q_name = $1 AND method = $2 AND args::text = $3::text
                    AND (locked_at IS NULL OR locked_at > current_timestamp - interval '#{max_lock_time} seconds')
                  LIMIT 1
-                 )
-                 
-                 UNION
-                 
-                 (
-                 SELECT 1
-                 FROM queue_classic_later_jobs
-                 WHERE q_name = $4 AND method = $5 AND args = $6::text
-                 LIMIT 1
-                 )
                )
-               AS x"
+             AS x"
 
-          result = QC.default_conn_adapter.execute(q, @queue, method, args.to_json, @queue, method, args.to_json)
-          result['count'].to_i == 0
-        end
+        result = QC.default_conn_adapter.execute(q, @queue, method, args.to_json)
+        result['count'].to_i == 0
       else
         true
       end
@@ -89,7 +77,7 @@ module QueueClassicPlus
     end
 
     def self.restart_in(time, remaining_retries, *args)
-      queue.enqueue_in_with_custom(time, "#{self.to_s}._perform", {'remaining_retries' => remaining_retries}, *args)
+      queue.enqueue_retry_in(time, "#{self.to_s}._perform", remaining_retries, *args)
     end
 
     def self.do(*args)
@@ -111,17 +99,36 @@ module QueueClassicPlus
     end
 
     def self.librato_key
-      (self.name || "").underscore.gsub(/\//, ".")
+      Inflector.underscore(self.name || "").gsub(/\//, ".")
     end
 
     def self.transaction(options = {}, &block)
-      ActiveRecord::Base.transaction(options, &block)
+      if defined?(ActiveRecord)
+        # If ActiveRecord is loaded, we use it's own transaction mechanisn since
+        # it has slightly different semanctics for rollback.
+        ActiveRecord::Base.transaction(options, &block)
+      else
+        begin
+          execute "BEGIN"
+          block.call
+        rescue
+          execute "ROLLBACK"
+          raise
+        end
+
+        execute "COMMIT"
+      end
     end
 
     # Debugging
     def self.list
       q = "SELECT * FROM queue_classic_jobs WHERE q_name = '#{@queue}'"
-      ActiveRecord::Base.connection.execute(q).to_a
+      execute q
+    end
+
+    private
+    def self.execute(sql, *args)
+      QC.default_conn_adapter.execute(sql, *args)
     end
   end
 end

data/lib/queue_classic_plus/inflector.rb

@@ -0,0 +1,15 @@
+module QueueClassicPlus
+  module Inflector
+    # Storngly inspired by
+    # https://github.com/rails/rails/blob/master/activesupport/lib/active_support/inflector/methods.rb#L91
+    def self.underscore(camel_cased_word)
+      return camel_cased_word unless camel_cased_word =~ /[A-Z-]|::/
+      word = camel_cased_word.to_s.gsub(/::/, '/')
+      word.gsub!(/([A-Z\d]+)([A-Z][a-z])/,'\1_\2')
+      word.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
+      word.tr!("-", "_")
+      word.downcase!
+      word
+    end
+  end
+end

data/lib/queue_classic_plus/inheritable_attr.rb

@@ -0,0 +1,35 @@
+module QueueClassicPlus
+  # From https://github.com/apotonick/uber/blob/master/lib/uber/inheritable_attr.rb which is MIT license
+  module InheritableAttribute
+    def inheritable_attr(name)
+      instance_eval %Q{
+        def #{name}=(v)
+          @#{name} = v
+        end
+        def #{name}
+          return @#{name} if instance_variable_defined?(:@#{name})
+          @#{name} = InheritableAttribute.inherit_for(self, :#{name})
+        end
+      }
+    end
+
+    def self.inherit_for(klass, name)
+      return unless klass.superclass.respond_to?(name)
+
+      value = klass.superclass.send(name) # could be nil.
+      Clone.(value) # this could be dynamic, allowing other inheritance strategies.
+    end
+
+    class Clone
+      # The second argument allows injecting more types.
+      def self.call(value, uncloneable=uncloneable())
+        uncloneable.each { |klass| return value if value.kind_of?(klass) }
+        value.clone
+      end
+
+      def self.uncloneable
+        [Symbol, TrueClass, FalseClass, NilClass]
+      end
+    end
+  end
+end

data/lib/queue_classic_plus/new_relic.rb

@@ -0,0 +1,35 @@
+require 'new_relic/agent/method_tracer'
+
+QueueClassicPlus::Base.class_eval do
+  class << self
+    include NewRelic::Agent::Instrumentation::ControllerInstrumentation
+
+    def new_relic_key
+      "Custom/QueueClassicPlus/#{librato_key}"
+    end
+
+    def _perform_with_new_relic(*args)
+      opts = {
+        name: 'perform',
+        class_name: self.name,
+        category: 'OtherTransaction/QueueClassicPlus',
+      }
+
+      perform_action_with_newrelic_trace(opts) do
+        if NewRelic::Agent.config[:'queue_classic_plus.capture_params']
+          NewRelic::Agent.add_custom_parameters(job_arguments: args)
+        end
+        _perform_without_new_relic *args
+      end
+    end
+
+    alias_method_chain :_perform, :new_relic
+  end
+end
+
+QueueClassicPlus::CustomWorker.class_eval do
+  def initialize(*)
+    ::NewRelic::Agent.manual_start
+    super
+  end
+end

data/lib/queue_classic_plus/queue_classic/queue.rb

@@ -0,0 +1,11 @@
+module QC
+  class Queue
+    def enqueue_retry_in(seconds, method, remaining_retries, *args)
+      QC.log_yield(:measure => 'queue.enqueue') do
+        s = "INSERT INTO #{TABLE_NAME} (q_name, method, args, scheduled_at, remaining_retries)
+             VALUES ($1, $2, $3, now() + interval '#{seconds.to_i} seconds', $4)"
+        res = conn_adapter.execute(s, name, method, JSON.dump(args), remaining_retries)
+      end
+    end
+  end
+end

data/lib/queue_classic_plus/update_metrics.rb

@@ -1,62 +1,83 @@
 module QueueClassicPlus
   module UpdateMetrics
     def self.update
-      {
-        "queued" => "queue_classic_jobs",
-        "scheduled" => "queue_classic_later_jobs",
-      }.each do |type, table|
-        next unless ActiveRecord::Base.connection.table_exists?(table)
-        q = "SELECT q_name, COUNT(1) FROM #{table} GROUP BY q_name"
-        results = execute(q)
-
-        # Log individual queue sizes
-        results.each do |h|
-          Metrics.measure("qc.jobs_#{type}", h.fetch('count').to_i, source: h.fetch('q_name'))
+      metrics.each do |name, values|
+        if values.respond_to?(:each)
+          values.each do |hash|
+            hash.to_a.each do |(source, count)|
+              Metrics.measure("qc.#{name}", count, source: source)
+            end
+          end
+        else
+          Metrics.measure("qc.#{name}", values)
         end
       end
+    end
 
-      # Log oldest locked_at and created_at
-      ['locked_at', 'created_at'].each do |column|
-        age = max_age(column)
-        Metrics.measure("qc.max_#{column}", age)
-      end
+    def self.metrics
+      {
+        jobs_queued: jobs_queued,
+        jobs_scheduled: jobs_scheduled,
+        max_created_at: max_age("created_at", "created_at = scheduled_at"),
+        max_locked_at: max_age("locked_at", "locked_at IS NOT NULL"),
+        "max_created_at.unlocked" => max_age("locked_at", "locked_at IS NULL"),
+        "jobs_delayed.lag" => max_age("scheduled_at"),
+        "jobs_delayed.late_count" => late_count,
+      }
+    end
 
-      # Log oldes unlocked jobs
-      age = max_age("created_at", "locked_at IS NULL")
-      Metrics.measure("qc.max_created_at.unlocked", age)
+    def self.jobs_queued
+      sql_group_count "SELECT q_name AS group, COUNT(1)
+              FROM queue_classic_jobs
+              WHERE scheduled_at <= NOW() GROUP BY q_name"
+    end
 
-      if ActiveRecord::Base.connection.table_exists?('queue_classic_later_jobs')
-        lag = execute("SELECT MAX(EXTRACT(EPOCH FROM now() - not_before)) AS lag 
-                FROM queue_classic_later_jobs").first
-                lag = lag ? lag['lag'] : 0
+    def self.jobs_scheduled
+      sql_group_count "SELECT q_name AS group, COUNT(1)
+              FROM queue_classic_jobs
+              WHERE scheduled_at > NOW() GROUP BY q_name"
+    end
 
-        Metrics.measure("qc.jobs_delayed.lag", lag.to_f)
+    def self.late_count
+      nb_late = execute("SELECT COUNT(1)
+         FROM queue_classic_jobs
+         WHERE scheduled_at < NOW() AND #{not_failed}")
+      nb_late ? Integer(nb_late['count']) : 0
+    end
 
-        nb_late = execute("SELECT COUNT(1) 
-           FROM queue_classic_later_jobs 
-           WHERE not_before < NOW()").first
-        nb_late = nb_late ? nb_late['count'] : 0
+    private
 
-        Metrics.measure("qc.jobs_delayed.late_count", nb_late.to_i)
+    def self.sql_group_count(sql)
+      results = execute(sql)
+      results = [results] if Hash === results
+      Array(results).map do |h|
+        {
+          h.fetch("group") => Integer(h.fetch('count'))
+        }
       end
     end
 
-    private
     def self.max_age(column, *conditions)
-      conditions.unshift "q_name != '#{::QueueClassicPlus::CustomWorker::FailedQueue.name}'"
+      conditions.unshift not_failed
+      conditions.unshift "scheduled_at <= NOW()"
 
-      q = "SELECT EXTRACT(EPOCH FROM now() - #{column}) AS age_in_seconds 
+      q = "SELECT EXTRACT(EPOCH FROM now() - #{column}) AS age_in_seconds
            FROM queue_classic_jobs
            WHERE #{conditions.join(" AND ")}
            ORDER BY age_in_seconds DESC
+           LIMIT 1
            "
-       age_info = execute(q).first
+       age_info = execute(q)
 
        age_info ? age_info['age_in_seconds'].to_i : 0
     end
 
     def self.execute(q)
-      ActiveRecord::Base.connection.execute(q)
+      QC.default_conn_adapter.execute(q)
+    end
+
+    def self.not_failed
+      "q_name != '#{::QueueClassicPlus::CustomWorker::FailedQueue.name}'"
     end
   end
 end

data/lib/queue_classic_plus/version.rb

@@ -1,3 +1,3 @@
 module QueueClassicPlus
-  VERSION = "0.0.2"
+  VERSION = "1.0.0.alpha2"
 end

data/lib/queue_classic_plus/worker.rb

@@ -1,18 +1,12 @@
 module QueueClassicPlus
   class CustomWorker < QC::Worker
-    class QueueClassicJob < ActiveRecord::Base
-    end
-
     BACKOFF_WIDTH = 10
     FailedQueue = QC::Queue.new("failed_jobs")
 
     def enqueue_failed(job, e)
-      ActiveRecord::Base.transaction do
-        FailedQueue.enqueue(job[:method], *job[:args])
-        new_job = QueueClassicJob.order(:id).where(q_name: 'failed_jobs').last
-        new_job.last_error = if e.backtrace then ([e.message] + e.backtrace ).join("\n") else e.message end
-        new_job.save!
-      end
+      sql = "INSERT INTO #{QC::TABLE_NAME} (q_name, method, args, last_error) VALUES ('failed_jobs', $1, $2, $3)"
+      last_error = e.backtrace ? ([e.message] + e.backtrace ).join("\n") : e.message
+      QC.default_conn_adapter.execute sql, job[:method], JSON.dump(job[:args]), last_error
 
       QueueClassicPlus.exception_handler.call(e, job)
       Metrics.increment("qc.errors", source: @q_name)
@@ -22,11 +16,9 @@ module QueueClassicPlus
       QueueClassicPlus.logger.info "Handling exception #{e.message} for job #{job[:id]}"
       klass = job_klass(job)
 
-      model = QueueClassicJob.find(job[:id])
-
       # The mailers doesn't have a retries_on?
       if klass && klass.respond_to?(:retries_on?) && klass.retries_on?(e)
-        remaining_retries = model.remaining_retries || klass.max_retries
+        remaining_retries = job[:remaining_retries] || klass.max_retries
         remaining_retries -= 1
 
         if remaining_retries > 0
@@ -39,7 +31,8 @@ module QueueClassicPlus
       else
         enqueue_failed(job, e)
       end
-      model.destroy
+
+      FailedQueue.delete(job[:id])
     end
 
     private

data/queue_classic_plus.gemspec

@@ -18,11 +18,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "queue_classic"
-  spec.add_dependency "queue_classic-later"
-  spec.add_dependency "activerecord", "> 3.0"
-  spec.add_dependency "activesupport", "> 3.0"
-  spec.add_dependency "with_advisory_lock"
+  spec.add_dependency "queue_classic", ">= 3.1.0"
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake"
 end

data/spec/base_spec.rb

@@ -17,9 +17,8 @@ describe QueueClassicPlus::Base do
 
       it "does allow multiple enqueues if something got locked for too long" do
         subject.do
-        ActiveRecord::Base.connection.execute "
-          UPDATE queue_classic_jobs SET locked_at = '#{1.day.ago.to_s}' WHERE q_name = 'test'
-        "
+        one_day_ago = Time.now - 60*60*24
+        execute "UPDATE queue_classic_jobs SET locked_at = '#{one_day_ago}' WHERE q_name = 'test'"
         subject.do
         subject.should have_queue_size_of(2)
       end
@@ -40,8 +39,8 @@ describe QueueClassicPlus::Base do
       end
 
       it "calls perform in a transaction" do
-        ActiveRecord::Base.should_receive(:transaction).and_call_original
-        subject._perform 
+        QueueClassicPlus::Base.should_receive(:transaction).and_call_original
+        subject._perform
       end
 
       it "measures the time" do
@@ -62,8 +61,8 @@ describe QueueClassicPlus::Base do
       end
 
       it "calls perform outside of a transaction" do
-        ActiveRecord::Base.should_not_receive(:transaction)
-        subject._perform 
+        QueueClassicPlus::Base.should_not_receive(:transaction)
+        subject._perform
       end
     end
 

data/spec/helpers.rb

@@ -0,0 +1,9 @@
+module QcHelpers
+  def execute(sql, *args)
+    QC.default_conn_adapter.execute(sql, *args)
+  end
+
+  def find_job(id)
+    execute("SELECT * FROM #{QC::TABLE_NAME} WHERE id = $1", id)
+  end
+end

data/spec/inflector_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+module QueueClassicPlus
+  describe Inflector do
+    describe ".underscore" do
+      {
+        "foo" => "foo",
+        "Foo" => "foo",
+        "FooBar" => "foo_bar",
+        "Foo::Bar" => "foo/bar"
+      }.each do |word, expected|
+        it "converst #{word} to #{expected}" do
+          expect(described_class.underscore(word)).to eq(expected)
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -3,29 +3,22 @@ require 'pg'
 require 'timecop'
 require 'queue_classic_matchers'
 require_relative './sample_jobs'
+require_relative './helpers'
+require 'pry'
+
+ENV["DATABASE_URL"] ||= "postgres:///queue_classic_plus_test"
 
 RSpec.configure do |config|
-  config.before(:suite) do
-    ActiveRecord::Base.establish_connection(
-      :adapter  => "postgresql",
-      :username => "postgres",
-      :database => "queue_classic_plus_test",
-      :host => 'localhost',
-    )
+  config.include QcHelpers
 
-    ActiveRecord::Base.connection.execute "drop schema public cascade; create schema public;"
+  config.before(:suite) do
+    QC.default_conn_adapter.execute "drop schema public cascade; create schema public;"
 
-    QC.default_conn_adapter = QC::ConnAdapter.new(ActiveRecord::Base.connection.raw_connection)
     QC::Setup.create
-    QC::Later::Setup.create
     QueueClassicPlus.migrate
   end
 
   config.before(:each) do
-    tables = ActiveRecord::Base.connection.tables.select do |table|
-      table != "schema_migrations"
-    end
-    ActiveRecord::Base.connection.execute("TRUNCATE #{tables.join(', ')} CASCADE") unless tables.empty?
-
+    QC.default_conn_adapter.execute "TRUNCATE queue_classic_jobs;"
   end
 end

data/spec/update_metrics_spec.rb

@@ -0,0 +1,125 @@
+require 'spec_helper'
+
+describe QueueClassicPlus::UpdateMetrics do
+  describe ".update" do
+    it "works" do
+      QC.enqueue "puts"
+
+      expect(QueueClassicPlus::Metrics).to(receive(:measure)).at_least(1).times do |metric, value, options|
+        expect(metric).to_not be_nil
+        expect(value).to_not be_nil
+      end
+      described_class.update
+    end
+  end
+
+  describe ".metrics" do
+    subject { described_class.metrics }
+
+    before do
+      QC.enqueue "puts"
+      QC.enqueue "puts", 2
+      QC.enqueue_in 60, "puts", 2
+    end
+
+    context "jobs_queued" do
+      it "returns the number of jobs group per queue" do
+        expect(subject[:jobs_queued]).to eq([{"default" => 2}])
+      end
+    end
+
+    context "jobs_scheduled" do
+      it "returns the number of jobs group per queue" do
+        expect(subject[:jobs_scheduled]).to eq([{"default" => 1}])
+      end
+    end
+
+    context "max_locked_at" do
+      it "zero if nothing is locked" do
+        max = subject[:max_locked_at]
+        expect(max).to eq(0)
+      end
+
+      it "returns the age of the oldest lock" do
+        execute "UPDATE queue_classic_jobs SET locked_at = '#{Time.now - 60}'"
+
+        max = subject[:max_locked_at]
+        expect(59..61).to include(max)
+      end
+
+      context 'scheduled jobs' do
+        it 'reports the correct max_locked_at' do
+          execute "UPDATE queue_classic_jobs SET locked_at = '#{Time.now - 30}', scheduled_at = '#{Time.now - 60}', created_at = '#{Time.now - 5*60}'"
+
+          expect(subject[:max_locked_at]).to be_within(1).of(30)
+        end
+      end
+    end
+
+    context "max_created_at" do
+      it "returns a small positive value" do
+        max = subject[:max_created_at]
+        expect(0..0.2).to include(max)
+      end
+
+      context 'scheduled jobs' do
+        it "ignores jobs schedule in the future" do
+          execute "UPDATE queue_classic_jobs SET created_at = '#{Time.now - 60}', scheduled_at = '#{Time.now + 60}'"
+
+          max = subject[:max_created_at]
+          expect(0..0.2).to include(max)
+        end
+
+        it 'reports time only for jobs that were never scheduled for future' do
+          execute "DELETE FROM queue_classic_jobs"
+          QC.enqueue 'puts'
+          QC.enqueue_in 1, 'puts'
+          one_min_ago = Time.now - 60
+          execute "UPDATE queue_classic_jobs SET created_at = '#{one_min_ago}', scheduled_at = '#{one_min_ago}'"
+          expect(subject[:max_created_at]).to be_within(1).of(60)
+        end
+      end
+    end
+
+    context "max_created_at.unlocked" do
+      it "ignores lock jobs" do
+        execute "UPDATE queue_classic_jobs SET locked_at = '#{Time.now - 60}', created_at = '#{Time.now - 2*60}'"
+
+        max = subject["max_created_at.unlocked"]
+        expect(max).to eq(0)
+      end
+    end
+
+    context "jobs_delayed.lag" do
+      it "returns the maximum different between the scheduled time and now" do
+        execute "UPDATE queue_classic_jobs SET scheduled_at = '#{Time.now - 60}'"
+
+        lag = subject["jobs_delayed.lag"]
+        expect(lag).to be_within(1.0).of(60)
+      end
+
+      it "ignores jobs scheduled in the future" do
+        execute "UPDATE queue_classic_jobs SET scheduled_at = '#{Time.now + 60}'"
+
+        lag = subject["jobs_delayed.lag"]
+        expect(lag).to eq(0)
+      end
+
+      it "ignores the failed queue" do
+        execute "UPDATE queue_classic_jobs SET scheduled_at = '#{Time.now - 60}', q_name = 'failed_jobs'"
+
+        lag = subject["jobs_delayed.lag"]
+        expect(lag).to eq(0)
+      end
+    end
+
+    context "jobs_delayed.late_count" do
+      it "returns the jobs that a created in the future" do
+        count = subject["jobs_delayed.late_count"]
+        # All jobs are always late because enqueue sets the value of
+        # scheduled_at to now() for normal jobs
+        expect(count).to eq(2)
+      end
+    end
+  end
+end

data/spec/worker_spec.rb

@@ -1,12 +1,6 @@
 require 'spec_helper'
 
 describe QueueClassicPlus::CustomWorker do
-  class QueueClassicLaterJob < ActiveRecord::Base
-  end
-
-  class QueueClassicJob < ActiveRecord::Base
-  end
-
   let(:failed_queue) { described_class::FailedQueue }
 
   context "failure" do
@@ -21,7 +15,9 @@ describe QueueClassicPlus::CustomWorker do
       job = failed_queue.lock
       job[:method].should == "Kerklfadsjflaksj"
       job[:args].should == [1, 2, 3]
-      QueueClassicJob.last.last_error.should be_present
+      full_job = find_job(job[:id])
+
+      full_job['last_error'].should_not be_nil
     end
 
     it "records normal errors" do
@@ -35,7 +31,7 @@ describe QueueClassicPlus::CustomWorker do
   context "retry" do
     let(:job_type) { Jobs::Tests::LockedTestJob }
     let(:worker) { described_class.new q_name: job_type.queue.name }
-    let(:enqueue_expected_ts) { described_class::BACKOFF_WIDTH.seconds.from_now }
+    let(:enqueue_expected_ts) { Time.now + described_class::BACKOFF_WIDTH }
 
     before do
       job_type.skip_transaction!
@@ -51,16 +47,14 @@ describe QueueClassicPlus::CustomWorker do
       QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries'].should be_nil
 
       Timecop.freeze do
-        expect do
-          worker.work
-        end.to change_queue_size_of(job_type).by(-1)
+        worker.work
 
         failed_queue.count.should == 0 # not enqueued on Failed
-        Jobs::Tests::LockedTestJob.should have_scheduled(true).at(Time.now + described_class::BACKOFF_WIDTH.seconds.to_i) # should have scheduled a retry for later
+        QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries'].should eq "4"
+        Jobs::Tests::LockedTestJob.should have_scheduled(true).at(Time.now + described_class::BACKOFF_WIDTH) # should have scheduled a retry for later
       end
 
-      Timecop.freeze(Time.now + (described_class::BACKOFF_WIDTH.seconds.to_i * 2)) do
-        QC::Later.tick(true)
+      Timecop.freeze(Time.now + (described_class::BACKOFF_WIDTH * 2)) do
         # the job should be re-enqueued with a decremented retry count
         jobs = QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true])
         jobs.size.should == 1
@@ -82,46 +76,11 @@ describe QueueClassicPlus::CustomWorker do
       QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries'].should be_nil
 
       Timecop.freeze do
-        expect do
-          worker.work
-        end.to change_queue_size_of(job_type).by(-1)
+        worker.work
 
+        QueueClassicMatchers::QueueClassicRspec.find_by_args('failed_jobs', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries'].should be_nil
         failed_queue.count.should == 1 # not enqueued on Failed
-        Jobs::Tests::LockedTestJob.should_not have_scheduled(true).at(Time.now + described_class::BACKOFF_WIDTH.seconds.to_i) # should have scheduled a retry for later
-      end
-    end
-  end
-
-  context "enqueuing during a retry" do
-    let(:job_type) { Jobs::Tests::LockedTestJob }
-    let(:worker) { described_class.new q_name: job_type.queue.name }
-    let(:enqueue_expected_ts) { described_class::BACKOFF_WIDTH.seconds.from_now }
-
-    before do
-      job_type.max_retries = 5
-      job_type.skip_transaction!
-    end
-
-    it "does not enqueue in main queue while retrying" do
-      expect do
-        job_type.enqueue_perform(true)
-      end.to change_queue_size_of(job_type).by(1)
-
-      Jobs::Tests::LockedTestJob.should have_queue_size_of(1)
-      failed_queue.count.should == 0
-      QueueClassicMatchers::QueueClassicRspec.find_by_args('low', 'Jobs::Tests::LockedTestJob._perform', [true]).first['remaining_retries'].should be_nil
-
-      Timecop.freeze do
-        expect do
-          worker.work
-        end.to change_queue_size_of(job_type).by(-1)
-
-        failed_queue.count.should == 0 # not enqueued on Failed
-        Jobs::Tests::LockedTestJob.should have_scheduled(true).at(Time.now + described_class::BACKOFF_WIDTH.seconds.to_i) # should have scheduled a retry for later
-
-        expect do
-          job_type.enqueue_perform(true)
-        end.to change_queue_size_of(job_type).by(0)
+        Jobs::Tests::LockedTestJob.should_not have_scheduled(true).at(Time.now + described_class::BACKOFF_WIDTH) # should have scheduled a retry for later
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: queue_classic_plus
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 1.0.0.alpha2
 platform: ruby
 authors:
 - Simon Mathieu
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-29 00:00:00.000000000 Z
+date: 2015-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: queue_classic
@@ -18,70 +18,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: queue_classic-later
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: activerecord
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: with_advisory_lock
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.1.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -122,13 +66,19 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".rvmrc"
+- ".travis.yml"
 - Gemfile
+- Guardfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - lib/queue_classic_plus.rb
 - lib/queue_classic_plus/base.rb
+- lib/queue_classic_plus/inflector.rb
+- lib/queue_classic_plus/inheritable_attr.rb
 - lib/queue_classic_plus/metrics.rb
+- lib/queue_classic_plus/new_relic.rb
+- lib/queue_classic_plus/queue_classic/queue.rb
 - lib/queue_classic_plus/railtie.rb
 - lib/queue_classic_plus/tasks/work.rake
 - lib/queue_classic_plus/update_metrics.rb
@@ -139,8 +89,11 @@ files:
 - lib/rails/generators/qc_plus_job/templates/job_spec.rb.erb
 - queue_classic_plus.gemspec
 - spec/base_spec.rb
+- spec/helpers.rb
+- spec/inflector_spec.rb
 - spec/sample_jobs.rb
 - spec/spec_helper.rb
+- spec/update_metrics_spec.rb
 - spec/worker_spec.rb
 homepage: ''
 licenses:
@@ -157,9 +110,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.2.2
@@ -168,6 +121,9 @@ specification_version: 4
 summary: Useful extras for Queue Classic
 test_files:
 - spec/base_spec.rb
+- spec/helpers.rb
+- spec/inflector_spec.rb
 - spec/sample_jobs.rb
 - spec/spec_helper.rb
+- spec/update_metrics_spec.rb
 - spec/worker_spec.rb