queue_classic_plus 1.0.0.alpha2 → 4.0.0.alpha8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f5b1934a03303e576a999e71f978a68da13f0298
-  data.tar.gz: d1de5250aa7b2ddfe4114043d87e5a95986145f8
+SHA256:
+  metadata.gz: 69af5e1cbbcc08c7cd0acad4c0f06e112a9fff09756a06c9ae2ffd80a71ab39a
+  data.tar.gz: cd3cc486050b9de66397099f81e0277aac83f5469bb925456527b331dec85a6a
 SHA512:
-  metadata.gz: ebadc9325c0307789f6a6216047a2c2cfe240d4364b613ee9eefc9a479224a9196813146b90ed251124cd5ad7a4966556c8fca8e1b48a490cfd806bf95124ff3
-  data.tar.gz: 619c736ed7b74f94841328acfa27926e2799fecc6589974d9b79de236c1f396ef126d1e80205df223e626536b69c1c0a834014e09e190b000635e571c2a8e677
+  metadata.gz: 8d3980cb5a576ff5681812a2e5d7497c35ff98dc84934778f4dd3c867643e10e9d87a38499bad6850cc72d1c4cd519a91a2509011f8ad916e78b10be3c2e076e
+  data.tar.gz: c31d0baa284b76731b268e29ab21b3ae0e679f7404345d3b449fb755a34df11100d6f6790a6f9e0dfb373ed908a8fa365575d88184f96d28a079270be5e67605

data/.circleci/config.yml

@@ -0,0 +1,67 @@
+version: 2.1
+
+jobs:
+  test:
+    docker:
+      - image: circleci/ruby:2.7.4-node
+        auth:
+          username: $DOCKERHUB_USERNAME
+          password: $DOCKERHUB_TOKEN
+        environment:
+          DATABASE_URL: postgres://circleci:circleci@127.0.0.1:5432/queue_classic_plus_test
+      - image: circleci/postgres:9.6.6-alpine
+        auth:
+          username: $DOCKERHUB_USERNAME
+          password: $DOCKERHUB_TOKEN
+        environment:
+          POSTGRES_USER: circleci
+          POSTGRES_PASSWORD: circleci
+          POSTGRES_DB: queue_classic_plus_test
+    steps:
+      - checkout
+      - run:
+          name: run tests
+          command: |
+            bundle check --path=vendor/bundle || bundle install --path=vendor/bundle --jobs=4 --retry=3
+            bundle exec rspec
+
+  push_to_rubygems:
+    docker:
+      - image: circleci/ruby:2.7.4
+        auth:
+          username: $DOCKERHUB_USERNAME
+          password: $DOCKERHUB_TOKEN
+    steps:
+      - checkout
+      - run:
+          name: Create .gem/credentials file
+          command: |
+            mkdir ~/.gem
+            echo "---
+              :rubygems_api_key: $RUBYGEMS_API_KEY
+            " > ~/.gem/credentials
+            chmod 600 ~/.gem/credentials
+      - run:
+          name: Release to rubygems
+          command: |
+            gem build queue_classic_plus
+            gem push queue_classic_plus-*.gem
+
+workflows:
+  version: 2
+  gem_release:
+    jobs:
+      - test:
+          context:
+            - DockerHub
+
+      - push_to_rubygems:
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v.*/
+          context:
+            - DockerHub

data/.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "13:00"
+  open-pull-requests-limit: 10

data/.gitignore

@@ -1,6 +1,7 @@
 *.gem
 *.rbc
 .bundle
+.byebug_history
 .config
 .yardoc
 Gemfile.lock
@@ -9,6 +10,7 @@ _yardoc
 coverage
 doc/
 lib/bundler/man
+log/
 pkg
 rdoc
 spec/reports
@@ -21,3 +23,4 @@ tmp
 *.a
 mkmf.log
 tags
+.project

data/Gemfile

@@ -12,6 +12,10 @@ group :development do
 end
 
 group :test do
+  gem 'byebug'
+  gem 'rake'
   gem 'rspec'
   gem 'timecop'
+  gem 'newrelic_rpm'
+  gem 'ddtrace'
 end

data/README.md

@@ -1,12 +1,12 @@
 # QueueClassicPlus
 
-[![Build Status](https://travis-ci.org/rainforestapp/queue_classic_plus.svg?branch=master)](https://travis-ci.org/rainforestapp/queue_classic_plus)
+[![rainforestapp](https://circleci.com/gh/rainforestapp/queue_classic_plus.svg?branch=master)](https://app.circleci.com/pipelines/github/rainforestapp/queue_classic_plus?branch=master)
 
-[QueueClassic](https://github.com/QueueClassic/queue_classic) is a simple Postgresql back DB queue. However, it's a little too simple to use it as the main queueing system of a medium to large app.
+[queue_classic](https://github.com/QueueClassic/queue_classic) is a simple Postgresql backed DB queue. However, it's a little too simple to use it as the main queueing system of a medium to large app. This was developed at [Rainforest QA](https://www.rainforestqa.com/).
 
 QueueClassicPlus adds many lacking features to QueueClassic.
 
-- Standard job format
+- Standardized job format
 - Retry on specific exceptions
 - Singleton jobs
 - Metrics
@@ -101,11 +101,21 @@ Jobs::UpdateMetrics.do 'type_a' # does not enqueues job since it's already queue
 Jobs::UpdateMetrics.do 'type_b' # enqueues job as the arguments are different.
 ```
 
+#### Transactions
+
+By default, all QueueClassicPlus jobs are executed in a PostgreSQL
+transaction. This decision was made because most jobs are usually
+pretty small and it's preferable to have all the benefits of the
+transaction. You can optionally specify a postgres statement timeout
+(in seconds) for all transactions with the environment variable
+`POSTGRES_STATEMENT_TIMEOUT`.
+
+You can disable this feature on a per job basis in the following way:
+
 ```ruby
 class Jobs::NoTransaction < QueueClassicPlus::Base
   # Don't run the perform method in a transaction
   skip_transaction!
-
   @queue = :low
 
   def self.perform(user_id)
@@ -114,19 +124,13 @@ class Jobs::NoTransaction < QueueClassicPlus::Base
 end
 ```
 
-#### Transaction
-
-By default, all QueueClassicPlus jobs are executed in a PostgreSQL transaction. This decision was made because most jobs are usually pretty small and it's preferable to have all the benefits of the transaction.
-
-You can disable this feature on a per job basis in the follwing way:
-
 ## Advanced configuration
 
 If you want to log exceptions in your favorite exception tracker. You can configured it like sso:
 
 ```ruby
 QueueClassicPlus.exception_handler = -> (exception, job) do
-  Raven.capture_exception(exception, extra: {job: job, env: ENV})
+  Sentry.capture_exception(exception, extra: { job: job, env: ENV })
 end
 ```
 
@@ -146,6 +150,12 @@ If you are using NewRelic and want to push performance data to it, you can add t
 require "queue_classic_plus/new_relic"
 ```
 
+To instrument DataDog monitoring add this to your QC initializer:
+
+```ruby
+require "queue_classic_plus/datadog"
+```
+
 ## Contributing
 
 1. Fork it ( https://github.com/[my-github-username]/queue_classic_plus/fork )

data/lib/queue_classic_plus/base.rb

@@ -10,11 +10,16 @@ module QueueClassicPlus
     inheritable_attr :skip_transaction
     inheritable_attr :retries_on
     inheritable_attr :max_retries
+    inheritable_attr :disable_retries
 
-    self.max_retries = 0
+    self.max_retries = 5
     self.retries_on = {}
+    self.disable_retries = false
 
     def self.retry!(on: RuntimeError, max: 5)
+      if self.disable_retries
+        raise 'retry! should not be used in conjuction with disable_retries!'
+      end
       Array(on).each {|e| self.retries_on[e] = true}
       self.max_retries = max
     end
@@ -23,6 +28,14 @@ module QueueClassicPlus
       self.retries_on[exception.class] || self.retries_on.keys.any? {|klass| exception.is_a? klass}
     end
 
+    def self.disable_retries!
+      unless self.retries_on.empty?
+        raise 'disable_retries! should not be enabled in conjunction with retry!'
+      end
+
+      self.disable_retries = true
+    end
+
     def self.lock!
       self.locked = true
     end
@@ -54,7 +67,7 @@ module QueueClassicPlus
                )
              AS x"
 
-        result = QC.default_conn_adapter.execute(q, @queue, method, args.to_json)
+        result = QC.default_conn_adapter.execute(q, @queue, method, JSON.dump(serialized(args)))
         result['count'].to_i == 0
       else
         true
@@ -63,7 +76,7 @@ module QueueClassicPlus
 
     def self.enqueue(method, *args)
       if can_enqueue?(method, *args)
-        queue.enqueue(method, *args)
+        queue.enqueue(method, *serialized(args))
       end
     end
 
@@ -73,11 +86,11 @@ module QueueClassicPlus
 
     def self.enqueue_perform_in(time, *args)
       raise "Can't enqueue in the future for locked jobs" if locked?
-      queue.enqueue_in(time, "#{self.to_s}._perform", *args)
+      queue.enqueue_in(time, "#{self.to_s}._perform", *serialized(args))
     end
 
     def self.restart_in(time, remaining_retries, *args)
-      queue.enqueue_retry_in(time, "#{self.to_s}._perform", remaining_retries, *args)
+      queue.enqueue_retry_in(time, "#{self.to_s}._perform", remaining_retries, *serialized(args))
     end
 
     def self.do(*args)
@@ -89,10 +102,13 @@ module QueueClassicPlus
     def self._perform(*args)
       Metrics.timing("qu_perform_time", source: librato_key) do
         if skip_transaction
-          perform *args
+          perform(*deserialized(args))
         else
           transaction do
-            perform *args
+            # .to_i defaults to 0, which means no timeout in postgres
+            timeout = ENV['POSTGRES_STATEMENT_TIMEOUT'].to_i * 1000
+            execute "SET LOCAL statement_timeout = #{timeout}"
+            perform(*deserialized(args))
           end
         end
       end
@@ -103,7 +119,7 @@ module QueueClassicPlus
     end
 
     def self.transaction(options = {}, &block)
-      if defined?(ActiveRecord)
+      if defined?(ActiveRecord) && ActiveRecord::Base.connected?
         # If ActiveRecord is loaded, we use it's own transaction mechanisn since
         # it has slightly different semanctics for rollback.
         ActiveRecord::Base.transaction(options, &block)
@@ -126,7 +142,26 @@ module QueueClassicPlus
       execute q
     end
 
+    protected
+
+    def self.serialized(args)
+      if defined?(Rails)
+        ActiveJob::Arguments.serialize(args)
+      else
+        args
+      end
+    end
+
+    def self.deserialized(args)
+      if defined?(Rails)
+        ActiveJob::Arguments.deserialize(args)
+      else
+        args
+      end
+    end
+
     private
+
     def self.execute(sql, *args)
       QC.default_conn_adapter.execute(sql, *args)
     end

data/lib/queue_classic_plus/datadog.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module QueueClassicDatadog
+  def _perform(*args)
+    Datadog.tracer.trace('qc.job', service_name: 'qc.job', resource: "#{name}#perform") do |_|
+      super
+    end
+  end
+
+  QueueClassicPlus::Base.singleton_class.send(:prepend, QueueClassicDatadog)
+end

data/lib/queue_classic_plus/inheritable_attr.rb

@@ -28,7 +28,9 @@ module QueueClassicPlus
       end
 
       def self.uncloneable
-        [Symbol, TrueClass, FalseClass, NilClass]
+        tmp = [Symbol, TrueClass, FalseClass, NilClass]
+        tmp += [Fixnum, Bignum] if RUBY_VERSION < '2.4.0'
+        tmp
       end
     end
   end

data/lib/queue_classic_plus/metrics.rb

@@ -7,7 +7,7 @@ module QueueClassicPlus
 
   class Metrics
     def self.timing(*args, &block)
-      provider.timing *args, &block
+      provider.timing(*args, &block)
     end
 
     def self.increment(*args)

data/lib/queue_classic_plus/new_relic.rb

@@ -1,30 +1,29 @@
 require 'new_relic/agent/method_tracer'
 
-QueueClassicPlus::Base.class_eval do
-  class << self
-    include NewRelic::Agent::Instrumentation::ControllerInstrumentation
+module QueueClassicNewRelic
+  include NewRelic::Agent::Instrumentation::ControllerInstrumentation
 
-    def new_relic_key
-      "Custom/QueueClassicPlus/#{librato_key}"
-    end
+  def new_relic_key
+    "Custom/QueueClassicPlus/#{librato_key}"
+  end
 
-    def _perform_with_new_relic(*args)
-      opts = {
-        name: 'perform',
-        class_name: self.name,
-        category: 'OtherTransaction/QueueClassicPlus',
-      }
+  def _perform(*args)
+    opts = {
+      name: 'perform',
+      class_name: self.name,
+      category: 'OtherTransaction/QueueClassicPlus',
+    }
 
-      perform_action_with_newrelic_trace(opts) do
-        if NewRelic::Agent.config[:'queue_classic_plus.capture_params']
-          NewRelic::Agent.add_custom_parameters(job_arguments: args)
-        end
-        _perform_without_new_relic *args
+    perform_action_with_newrelic_trace(opts) do
+      if NewRelic::Agent.config[:'queue_classic_plus.capture_params']
+        NewRelic::Agent.add_custom_parameters(job_arguments: args)
       end
-    end
 
-    alias_method_chain :_perform, :new_relic
+      super
+    end
   end
+
+  QueueClassicPlus::Base.singleton_class.send(:prepend, QueueClassicNewRelic)
 end
 
 QueueClassicPlus::CustomWorker.class_eval do

data/lib/queue_classic_plus/queue_classic/queue.rb

@@ -1,11 +1,54 @@
 module QC
   class Queue
+
     def enqueue_retry_in(seconds, method, remaining_retries, *args)
       QC.log_yield(:measure => 'queue.enqueue') do
-        s = "INSERT INTO #{TABLE_NAME} (q_name, method, args, scheduled_at, remaining_retries)
+        s = "INSERT INTO #{QC.table_name} (q_name, method, args, scheduled_at, remaining_retries)
              VALUES ($1, $2, $3, now() + interval '#{seconds.to_i} seconds', $4)"
-        res = conn_adapter.execute(s, name, method, JSON.dump(args), remaining_retries)
+
+        conn_adapter.execute(s, name, method, JSON.dump(args), remaining_retries)
       end
     end
+
+    def lock
+      QC.log_yield(:measure => 'queue.lock') do
+        s = <<~SQL
+          WITH selected_job AS (
+            SELECT id
+            FROM queue_classic_jobs
+            WHERE
+              locked_at IS NULL AND
+              q_name = $1 AND
+              scheduled_at <= now()
+            LIMIT 1
+            FOR NO KEY UPDATE SKIP LOCKED
+          )
+          UPDATE queue_classic_jobs
+          SET
+            locked_at = now(),
+            locked_by = pg_backend_pid()
+          FROM selected_job
+          WHERE queue_classic_jobs.id = selected_job.id
+          RETURNING *
+        SQL
+
+        if r = conn_adapter.execute(s, name)
+          {}.tap do |job|
+            job[:id] = r["id"]
+            job[:q_name] = r["q_name"]
+            job[:method] = r["method"]
+            job[:args] = JSON.parse(r["args"])
+            job[:remaining_retries] = r["remaining_retries"]&.to_s
+            if r["scheduled_at"]
+              # ActiveSupport may cast time strings to Time
+              job[:scheduled_at] = r["scheduled_at"].kind_of?(Time) ? r["scheduled_at"] : Time.parse(r["scheduled_at"])
+              ttl = Integer((Time.now - job[:scheduled_at]) * 1000)
+              QC.measure("time-to-lock=#{ttl}ms source=#{name}")
+            end
+          end
+        end
+      end
+    end
+
   end
-end
+end

data/lib/queue_classic_plus/tasks/work.rake

@@ -2,13 +2,26 @@ namespace :qc_plus do
   desc "Start a new worker for the (default or $QUEUE) queue"
   task :work  => :environment do
     puts "Starting up worker for queue #{ENV['QUEUE']}"
+
+    # ActiveRecord::RecordNotFound is ignored by Sentry by default,
+    # which shouldn't happen in background jobs.
+    if defined?(Sentry)
+      Sentry.init do |config|
+        config.excluded_exceptions = []
+        config.background_worker_threads = 0 if Gem::Version.new(Sentry::VERSION) >= Gem::Version.new('4.1.0')
+      end
+    elsif defined?(Raven)
+      Raven.configure do |config|
+        config.excluded_exceptions = []
+      end
+    end
+
     @worker = QueueClassicPlus::CustomWorker.new
 
     trap('INT') do
       $stderr.puts("Received INT. Shutting down.")
       if !@worker.running
-        $stderr.puts("Worker has stopped running. Exit.")
-        exit(1)
+        $stderr.puts("Worker has already stopped running.")
       end
       @worker.stop
     end
@@ -19,5 +32,6 @@ namespace :qc_plus do
     end
 
     @worker.start
+    $stderr.puts 'Shut down successfully'
   end
 end

data/lib/queue_classic_plus/version.rb

@@ -1,3 +1,3 @@
 module QueueClassicPlus
-  VERSION = "1.0.0.alpha2"
+  VERSION = '4.0.0.alpha8'.freeze
 end

data/lib/queue_classic_plus/worker.rb

@@ -1,47 +1,89 @@
+require 'pg'
+require 'queue_classic'
+
 module QueueClassicPlus
   class CustomWorker < QC::Worker
+    CONNECTION_ERRORS = [PG::UnableToSend, PG::ConnectionBad].freeze
     BACKOFF_WIDTH = 10
     FailedQueue = QC::Queue.new("failed_jobs")
 
-    def enqueue_failed(job, e)
-      sql = "INSERT INTO #{QC::TABLE_NAME} (q_name, method, args, last_error) VALUES ('failed_jobs', $1, $2, $3)"
-      last_error = e.backtrace ? ([e.message] + e.backtrace ).join("\n") : e.message
-      QC.default_conn_adapter.execute sql, job[:method], JSON.dump(job[:args]), last_error
+    def handle_failure(job, e)
+      QueueClassicPlus.logger.info "Handling exception #{e.class} - #{e.message} for job #{job[:id]}"
 
-      QueueClassicPlus.exception_handler.call(e, job)
-      Metrics.increment("qc.errors", source: @q_name)
-    end
+      force_retry = false
+      if connection_error?(e)
+        # If we've got here, unfortunately ActiveRecord's rollback mechanism may
+        # not have kicked in yet and we might be in a failed transaction. To be
+        # *absolutely* sure the retry/failure gets enqueued, we do a rollback
+        # just in case (and if we're not in a transaction it will be a no-op).
+        QueueClassicPlus.logger.info "Reset connection for job #{job[:id]}"
+        @conn_adapter.connection.reset
+        @conn_adapter.execute 'ROLLBACK'
 
-    def handle_failure(job, e)
-      QueueClassicPlus.logger.info "Handling exception #{e.message} for job #{job[:id]}"
-      klass = job_klass(job)
+        # We definitely want to retry because the connection was lost mid-task.
+        force_retry = true
+      end
 
+      @failed_job = job
+      @failed_job_args = failed_job_class ? failed_job_class.deserialized(job[:args]) : job[:args]
+
+      if force_retry && !(failed_job_class.respond_to?(:disable_retries) && failed_job_class.disable_retries)
+        Metrics.increment("qc.force_retry", source: @q_name)
+        retry_with_remaining(e)
       # The mailers doesn't have a retries_on?
-      if klass && klass.respond_to?(:retries_on?) && klass.retries_on?(e)
-        remaining_retries = job[:remaining_retries] || klass.max_retries
-        remaining_retries -= 1
-
-        if remaining_retries > 0
-          klass.restart_in((klass.max_retries - remaining_retries) * BACKOFF_WIDTH,
-                           remaining_retries,
-                           *job[:args])
-        else
-          enqueue_failed(job, e)
-        end
+    elsif failed_job_class && failed_job_class.respond_to?(:retries_on?) && failed_job_class.retries_on?(e)
+        Metrics.increment("qc.retry", source: @q_name)
+        retry_with_remaining(e)
       else
-        enqueue_failed(job, e)
+        enqueue_failed(e)
       end
 
-      FailedQueue.delete(job[:id])
+      FailedQueue.delete(@failed_job[:id])
     end
 
     private
-    def job_klass(job)
+
+    def retry_with_remaining(e)
+      if remaining_retries > 0
+        failed_job_class.restart_in(backoff, remaining_retries - 1, *@failed_job_args)
+      else
+        enqueue_failed(e)
+      end
+    end
+
+    def max_retries
+      failed_job_class.respond_to?(:max_retries) ? failed_job_class.max_retries : 5
+    end
+
+    def remaining_retries
+      (@failed_job[:remaining_retries] || max_retries).to_i
+    end
+
+    def failed_job_class
       begin
-        Object.const_get(job[:method].split('.')[0])
+        Object.const_get(@failed_job[:method].split('.')[0])
       rescue NameError
         nil
       end
     end
+
+    def backoff
+      (max_retries - (remaining_retries - 1)) * BACKOFF_WIDTH
+    end
+
+    def connection_error?(e)
+      CONNECTION_ERRORS.any? { |klass| e.kind_of? klass } ||
+        (e.respond_to?(:original_exception) &&
+         CONNECTION_ERRORS.any? { |klass| e.original_exception.kind_of? klass })
+    end
+
+    def enqueue_failed(e)
+      sql = "INSERT INTO #{QC.table_name} (q_name, method, args, last_error) VALUES ('failed_jobs', $1, $2, $3)"
+      last_error = e.backtrace ? ([e.message] + e.backtrace ).join("\n") : e.message
+      QC.default_conn_adapter.execute sql, @failed_job[:method], JSON.dump(@failed_job_args), last_error
+
+      QueueClassicPlus.exception_handler.call(e, @failed_job)
+      Metrics.increment("qc.errors", source: @q_name)
+    end
   end
 end

data/lib/queue_classic_plus.rb

@@ -14,19 +14,19 @@ module QueueClassicPlus
   require 'queue_classic_plus/railtie' if defined?(Rails)
 
   def self.migrate(c = QC::default_conn_adapter.connection)
-    conn = QC::ConnAdapter.new(c)
+    conn = QC::ConnAdapter.new(connection: c)
     conn.execute("ALTER TABLE queue_classic_jobs ADD COLUMN last_error TEXT")
     conn.execute("ALTER TABLE queue_classic_jobs ADD COLUMN remaining_retries INTEGER")
   end
 
   def self.demigrate(c = QC::default_conn_adapter.connection)
-    conn = QC::ConnAdapter.new(c)
+    conn = QC::ConnAdapter.new(connection: c)
     conn.execute("ALTER TABLE queue_classic_jobs DROP COLUMN last_error")
     conn.execute("ALTER TABLE queue_classic_jobs DROP COLUMN remaining_retries")
   end
 
   def self.exception_handler
-    @exception_handler ||= -> (exception, job) { nil }
+    @exception_handler ||= ->(exception, job) { nil }
   end
 
   def self.exception_handler=(handler)

data/queue_classic_plus.gemspec

@@ -18,7 +18,13 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "queue_classic", ">= 3.1.0"
-  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_dependency "queue_classic", "4.0.0.pre.alpha1"
+  if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3.0')
+    spec.add_development_dependency "bundler", "~> 1.6"
+  else
+    spec.add_development_dependency "bundler", "~> 2.0"
+  end
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "activerecord", "~> 6.0"
+  spec.add_development_dependency "activejob"
 end