backburner 1.3.1 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: ffcc49440304bb0d25a3710a3d94442fd1881f51
-  data.tar.gz: 2e9318efc9a04979135fecc8edfce4e8c8912386
+SHA256:
+  metadata.gz: 93c9b77199209b894ae963d9474a3234212225021537f99b9fee11433e73f6c4
+  data.tar.gz: 49e775e7a32ccc85a557f27ba89a146617553b45e0a252ddfe12f84d1e501706
 SHA512:
-  metadata.gz: caa6aa25343e41b2e8097be38d6b5679f304b9f417f2b0559e085caa2165660fc3ab132f5a920a49360faf16629b84b2d1b740ddf8332b0106df0437458a6f40
-  data.tar.gz: 457cb6f9ce73c900e0c54face79f78c8f0f6a89b1462580208f5afa9450f17c717fd7b2dd5701299addb334e260e55baa5c5360583867268c9f4ea921c94e38b
+  metadata.gz: f6e5cdea7a5e3092b8dfc61199ccabf1eb898b3eae4c5645171a7fecc415d7c1c467d1fd173d78732dee6c60bfd2f5928cb728a52e13269bbd951d6d304a2ceb
+  data.tar.gz: e0da0a58b32ea88704e33a4e19a03b3f3b1c8cb63428091743953112841e6b2fc1bf27d5de18d51a8777cad9ee45e069a02a0f9cc132fdf8a0e036bd0379e162

data/.travis.yml

@@ -2,6 +2,11 @@
 rvm:
   - 1.9.3
   - 2.0.0
+  - 2.1
+  - 2.2
+  - 2.3
+  - 2.4
+  - 2.5
   - rbx-2
 before_install:
   - curl -L https://github.com/kr/beanstalkd/archive/v1.9.tar.gz | tar xz -C /tmp
@@ -9,11 +14,13 @@ before_install:
   - make
   - ./beanstalkd &
   - cd $TRAVIS_BUILD_DIR
+  - gem update --system
+  - gem update bundler
 matrix:
   allow_failures:
     - rvm: rbx-2
+    - rvm: 2.0.0
 script:
-  - bundle install
   - bundle exec rake test
 gemfile: Gemfile
 notifications:

data/CHANGELOG.md

@@ -1,5 +1,24 @@
 # CHANGELOG
 
+## Version 1.6.0 (December 30 2021)
+
+* TBD (please help backfill)
+
+## Version 1.5.0 (September 10 2018)
+
+* TBD
+
+## Version 1.4.1 (June 10 2017)
+
+* Fix warning for constant ::Fixnum is deprecated (@amatsuda)
+
+## Version 1.4.0 (May 13 2017)
+
+* Fix unit tests to be more consistent (@eltone)
+* Ensure job supports body hash with symbol keys (@eltone)
+* Add support for custom serialization formats (@eltone)
+* Log the params when a job timeout occurs (@nathantsoi)
+
 ## Version 1.3.1 (April 21 2016)
 
 * Addition of thread-pool-based concurrency (@contentfree)

data/README.md

@@ -1,6 +1,6 @@
-# Backburner
+# Backburner [![Build Status](https://travis-ci.org/nesquena/backburner.svg?branch=master)](https://travis-ci.org/nesquena/backburner)
 
-Backburner is a [beanstalkd](http://kr.github.com/beanstalkd/)-powered job queue that can handle a very high volume of jobs.
+Backburner is a [beanstalkd](https://beanstalkd.github.io/)-powered job queue that can handle a very high volume of jobs.
 You create background jobs and place them on multiple work queues to be processed later.
 
 Processing background jobs reliably has never been easier than with beanstalkd and Backburner. This gem works with any ruby-based
@@ -34,7 +34,7 @@ The real question then is... "Why Beanstalk?".
 
 Illya has an excellent blog post
 [Scalable Work Queues with Beanstalk](http://www.igvita.com/2010/05/20/scalable-work-queues-with-beanstalk/) and
-Adam Wiggins posted [an excellent comparison](http://adam.heroku.com/past/2010/4/24/beanstalk_a_simple_and_fast_queueing_backend/).
+Adam Wiggins posted [an excellent comparison](http://adam.herokuapp.com/past/2010/4/24/beanstalk_a_simple_and_fast_queueing_backend/).
 
 You will quickly see that **beanstalkd** is an underrated but incredible project that is extremely well-suited as a job queue.
 Significantly better suited for this task than Redis or a database. Beanstalk is a simple,
@@ -65,7 +65,7 @@ In the end, **beanstalk is the ideal job queue** while also being ridiculously e
 
 ## Installation
 
-First, you probably want to [install beanstalkd](http://kr.github.com/beanstalkd/download.html), which powers the job queues.
+First, you probably want to [install beanstalkd](https://beanstalkd.github.io/download.html), which powers the job queues.
 Depending on your platform, this should be as simple as (for Ubuntu):
 
     $ sudo apt-get install beanstalkd
@@ -88,7 +88,7 @@ Backburner is extremely simple to setup. Just configure basic settings for backb
 
 ```ruby
 Backburner.configure do |config|
-  config.beanstalk_url       = ["beanstalk://127.0.0.1", "..."]
+  config.beanstalk_url       = "beanstalk://127.0.0.1"
   config.tube_namespace      = "some.app.production"
   config.namespace_separator = "."
   config.on_error            = lambda { |e| puts e }
@@ -102,6 +102,9 @@ Backburner.configure do |config|
   config.primary_queue       = "backburner-jobs"
   config.priority_labels     = { :custom => 50, :useless => 1000 }
   config.reserve_timeout     = nil
+  config.job_serializer_proc = lambda { |body| JSON.dump(body) }
+  config.job_parser_proc     = lambda { |body| JSON.parse(body) }
+
 end
 ```
 
@@ -109,7 +112,7 @@ The key options available are:
 
 | Option                | Description                                                          |
 | -----------------     | -------------------------------                                      |
-| `beanstalk_url`       | Address such as 'beanstalk://127.0.0.1' or an array of addresses.    |
+| `beanstalk_url`       | Address for beanstalkd connection i.e 'beanstalk://127.0.0.1'        |
 | `tube_namespace`      | Prefix used for all tubes related to this backburner queue.          |
 | `namespace_separator` | Separator used for namespace and queue name                          |
 | `on_error`            | Lambda invoked with the error whenever any job in the system fails.  |
@@ -123,6 +126,8 @@ The key options available are:
 | `primary_queue`       | Primary queue used for a job when an alternate queue is not given.   |
 | `priority_labels`     | Hash of named priority definitions for your app.                     |
 | `reserve_timeout`     | Duration to wait for work from a single server, or nil for forever.  |
+| `job_serializer_proc` | Lambda serializes a job body to a string to write to the task        |
+| `job_parser_proc`     | Lambda parses a task body string to a hash                           |
 
 ## Breaking Changes
 
@@ -158,10 +163,25 @@ class NewsletterJob
     1000 # most urgent priority is 0
   end
 
-  # optional, defaults to respond_timeout
+  # optional, defaults to respond_timeout in config
   def self.queue_respond_timeout
     300 # number of seconds before job times out, 0 to avoid timeout. NB: A timeout of 1 second will likely lead to race conditions between Backburner and beanstalkd and should be avoided
   end
+
+  # optional, defaults to retry_delay_proc in config
+  def self.queue_retry_delay_proc
+    lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 5) }
+  end
+
+  # optional, defaults to retry_delay in config
+  def self.queue_retry_delay
+    5
+  end
+
+  # optional, defaults to max_job_retries in config
+  def self.queue_max_job_retries
+    5
+  end
 end
 ```
 
@@ -508,6 +528,9 @@ end
 
 Now all backburner queue errors will appear on airbrake for deeper inspection.
 
+If you wish to retry a job without logging an error (for example when handling transient issues in a cloud or service oriented environment), 
+simply raise a `Backburner::Job::RetryJob` error.
+
 ### Logging
 
 Logging in backburner is rather simple. When a job is run, the log records that. When a job

data/backburner.gemspec

@@ -18,10 +18,9 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency 'beaneater', '~> 1.0'
   s.add_runtime_dependency 'dante', '> 0.1.5'
-  s.add_runtime_dependency 'concurrent-ruby', '~> 1.0.1'
+  s.add_runtime_dependency 'concurrent-ruby', '~> 1.0', '>= 1.0.1'
 
   s.add_development_dependency 'rake'
   s.add_development_dependency 'minitest', '3.2.0'
   s.add_development_dependency 'mocha'
-  s.add_development_dependency 'byebug'
 end

data/lib/backburner/configuration.rb

@@ -17,6 +17,8 @@ module Backburner
     attr_accessor :primary_queue       # the general queue
     attr_accessor :priority_labels     # priority labels
     attr_accessor :reserve_timeout     # duration to wait to reserve on a single server
+    attr_accessor :job_serializer_proc # proc to write the job body to a string
+    attr_accessor :job_parser_proc     # proc to parse a job body from a string
 
     def initialize
       @beanstalk_url       = "beanstalk://127.0.0.1"
@@ -34,6 +36,8 @@ module Backburner
       @primary_queue       = "backburner-jobs"
       @priority_labels     = PRIORITY_LABELS
       @reserve_timeout     = nil
+      @job_serializer_proc = lambda { |body| body.to_json }
+      @job_parser_proc     = lambda { |body| JSON.parse(body) }
     end
 
     def namespace_separator=(val)

data/lib/backburner/connection.rb

@@ -34,7 +34,7 @@ module Backburner
     def connected?
       begin
         !!(@beanstalk && @beanstalk.connection && @beanstalk.connection.connection && !@beanstalk.connection.connection.closed?) # Would be nice if beaneater provided a connected? method
-      rescue => e
+      rescue
         false
       end
     end

data/lib/backburner/helpers.rb

@@ -114,7 +114,7 @@ module Backburner
         resolve_priority(pri.queue_priority)
       elsif pri.is_a?(String) || pri.is_a?(Symbol) # named priority
         resolve_priority(Backburner.configuration.priority_labels[pri.to_sym])
-      elsif pri.is_a?(Fixnum) # numerical
+      elsif pri.is_a?(Integer) # numerical
         pri
       else # default
         Backburner.configuration.default_priority
@@ -131,12 +131,63 @@ module Backburner
     def resolve_respond_timeout(ttr)
       if ttr.respond_to?(:queue_respond_timeout)
         resolve_respond_timeout(ttr.queue_respond_timeout)
-      elsif ttr.is_a?(Fixnum) # numerical
+      elsif ttr.is_a?(Integer) # numerical
         ttr
       else # default
         Backburner.configuration.respond_timeout
       end
     end
 
+    # Resolves max retries based on the value given. Can be integer, a class or nothing
+    #
+    # @example
+    #  resolve_max_job_retries(5) => 5
+    #  resolve_max_job_retries(FooBar) => <queue max_job_retries>
+    #  resolve_max_job_retries(nil) => <default max_job_retries>
+    #
+    def resolve_max_job_retries(retries)
+      if retries.respond_to?(:queue_max_job_retries)
+        resolve_max_job_retries(retries.queue_max_job_retries)
+      elsif retries.is_a?(Integer) # numerical
+        retries
+      else # default
+        Backburner.configuration.max_job_retries
+      end
+    end
+
+    # Resolves retry delay based on the value given. Can be integer, a class or nothing
+    #
+    # @example
+    #  resolve_retry_delay(5) => 5
+    #  resolve_retry_delay(FooBar) => <queue retry_delay>
+    #  resolve_retry_delay(nil) => <default retry_delay>
+    #
+    def resolve_retry_delay(delay)
+      if delay.respond_to?(:queue_retry_delay)
+        resolve_retry_delay(delay.queue_retry_delay)
+      elsif delay.is_a?(Integer) # numerical
+        delay
+      else # default
+        Backburner.configuration.retry_delay
+      end
+    end
+
+    # Resolves retry delay proc based on the value given. Can be proc, a class or nothing
+    #
+    # @example
+    #  resolve_retry_delay_proc(proc) => proc
+    #  resolve_retry_delay_proc(FooBar) => <queue retry_delay_proc>
+    #  resolve_retry_delay_proc(nil) => <default retry_delay_proc>
+    #
+    def resolve_retry_delay_proc(proc)
+      if proc.respond_to?(:queue_retry_delay_proc)
+        resolve_retry_delay_proc(proc.queue_retry_delay_proc)
+      elsif proc.is_a?(Proc)
+        proc
+      else # default
+        Backburner.configuration.retry_delay_proc
+      end
+    end
+
   end # Helpers
 end # Backburner

data/lib/backburner/job.rb

@@ -7,6 +7,7 @@ module Backburner
     class JobTimeout < RuntimeError; end
     class JobNotFound < RuntimeError; end
     class JobFormatInvalid < RuntimeError; end
+    class RetryJob < RuntimeError; end
 
     attr_accessor :task, :body, :name, :args
 
@@ -21,8 +22,9 @@ module Backburner
     def initialize(task)
       @hooks = Backburner::Hooks
       @task = task
-      @body = task.body.is_a?(Hash) ? task.body : JSON.parse(task.body)
-      @name, @args = body["class"], body["args"]
+      @body = task.body.is_a?(Hash) ? task.body : Backburner.configuration.job_parser_proc.call(task.body)
+      @name = body["class"] || body[:class]
+      @args = body["args"] || body[:args]
     rescue => ex # Job was not valid format
       self.bury
       raise JobFormatInvalid, "Job body could not be parsed: #{ex.inspect}"
@@ -42,10 +44,10 @@ module Backburner
     #
     def process
       # Invoke before hook and stop if false
-      res = @hooks.invoke_hook_events(job_class, :before_perform, *args)
+      res = @hooks.invoke_hook_events(job_name, :before_perform, *args)
       return false unless res
       # Execute the job
-      @hooks.around_hook_events(job_class, :around_perform, *args) do
+      @hooks.around_hook_events(job_name, :around_perform, *args) do
         # We subtract one to ensure we timeout before beanstalkd does, except if:
         #  a) ttr == 0, to support never timing out
         #  b) ttr == 1, so that we don't accidentally set it to never time out
@@ -55,35 +57,50 @@ module Backburner
       end
       task.delete
       # Invoke after perform hook
-      @hooks.invoke_hook_events(job_class, :after_perform, *args)
+      @hooks.invoke_hook_events(job_name, :after_perform, *args)
     rescue => e
-      @hooks.invoke_hook_events(job_class, :on_failure, e, *args)
+      @hooks.invoke_hook_events(job_name, :on_failure, e, *args)
       raise e
     end
 
     def bury
-      @hooks.invoke_hook_events(job_class, :on_bury, *args)
+      @hooks.invoke_hook_events(job_name, :on_bury, *args)
       task.bury
     end
 
     def retry(count, delay)
-      @hooks.invoke_hook_events(job_class, :on_retry, count, delay, *args)
+      @hooks.invoke_hook_events(job_name, :on_retry, count, delay, *args)
       task.release(delay: delay)
     end
 
-    protected
-
     # Returns the class for the job handler
     #
     # @example
     #   job_class # => NewsletterSender
     #
     def job_class
-      handler = constantize(self.name) rescue nil
+      handler = try_job_class
       raise(JobNotFound, self.name) unless handler
       handler
     end
 
+    protected
+
+    # Attempts to return a constantized job name, otherwise reverts to the name string
+    #
+    # @example
+    #   job_name # => "SomeUnknownJob"
+    def job_name
+      handler = try_job_class
+      handler ? handler : self.name
+    end
+
+    def try_job_class
+      constantize(self.name)
+    rescue NameError
+      nil
+    end
+
     # Timeout job within specified block after given time.
     #
     # @example
@@ -93,7 +110,7 @@ module Backburner
       begin
         Timeout::timeout(secs) { yield }
       rescue Timeout::Error => e
-        raise JobTimeout, "#{name} hit #{secs}s timeout.\nbacktrace: #{e.backtrace}"
+        raise JobTimeout, "#{name}(#{(@args||[]).join(', ')}) hit #{secs}s timeout.\nbacktrace: #{e.backtrace}"
       end
     end
 

data/lib/backburner/logger.rb

@@ -10,7 +10,7 @@ module Backburner
     # Print out when a job is about to begin
     def log_job_begin(name, args)
       log_info "Work job #{name} with #{args.inspect}"
-      @job_started_at = Time.now
+      Thread.current[:job_started_at] = Time.now
     end
 
     # Print out when a job completed
@@ -24,7 +24,7 @@ module Backburner
 
     # Returns true if the job logging started
     def job_started_at
-      @job_started_at
+      Thread.current[:job_started_at]
     end
 
     # Print a message to stdout
@@ -50,4 +50,4 @@ module Backburner
       Backburner.configuration.logger
     end
   end
-end
+end

data/lib/backburner/queue.rb

@@ -4,6 +4,9 @@ module Backburner
       base.instance_variable_set(:@queue_name, nil)
       base.instance_variable_set(:@queue_priority, nil)
       base.instance_variable_set(:@queue_respond_timeout, nil)
+      base.instance_variable_set(:@queue_max_job_retries, nil)
+      base.instance_variable_set(:@queue_retry_delay, nil)
+      base.instance_variable_set(:@queue_retry_delay_proc, nil)
       base.instance_variable_set(:@queue_jobs_limit, nil)
       base.instance_variable_set(:@queue_garbage_limit, nil)
       base.instance_variable_set(:@queue_retry_limit, nil)
@@ -54,6 +57,48 @@ module Backburner
         end
       end
 
+      # Returns or assigns queue max_job_retries for this job
+      #
+      # @example
+      #   queue_max_job_retries 120
+      #   @klass.queue_max_job_retries # => 120
+      #
+      def queue_max_job_retries(delay=nil)
+        if delay
+          @queue_max_job_retries = delay
+        else # accessor
+          @queue_max_job_retries
+        end
+      end
+
+      # Returns or assigns queue retry_delay for this job
+      #
+      # @example
+      #   queue_retry_delay 120
+      #   @klass.queue_retry_delay # => 120
+      #
+      def queue_retry_delay(delay=nil)
+        if delay
+          @queue_retry_delay = delay
+        else # accessor
+          @queue_retry_delay
+        end
+      end
+
+      # Returns or assigns queue retry_delay_proc for this job
+      #
+      # @example
+      #   queue_retry_delay_proc lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 2) }
+      #   @klass.queue_retry_delay_proc # => lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 2) }
+      #
+      def queue_retry_delay_proc(proc=nil)
+        if proc
+          @queue_retry_delay_proc = proc
+        else # accessor
+          @queue_retry_delay_proc
+        end
+      end
+
       # Returns or assigns queue parallel active jobs limit (only ThreadsOnFork and Threading workers)
       #
       # @example

data/lib/backburner/version.rb

@@ -1,3 +1,3 @@
 module Backburner
-  VERSION = "1.3.1"
+  VERSION = "1.6.0"
 end

data/lib/backburner/worker.rb

@@ -39,7 +39,8 @@ module Backburner
         connection = Backburner::Connection.new(Backburner.configuration.beanstalk_url)
         connection.retryable do
           tube = connection.tubes[expand_tube_name(queue || job_class)]
-          response = tube.put(data.to_json, :pri => pri, :delay => delay, :ttr => ttr)
+          serialized_data = Backburner.configuration.job_serializer_proc.call(data)
+          response = tube.put(serialized_data, :pri => pri, :delay => delay, :ttr => ttr)
         end
         return nil unless Backburner::Hooks.invoke_hook_events(job_class, :after_enqueue, *args)
       ensure
@@ -139,7 +140,7 @@ module Backburner
     rescue Backburner::Job::JobFormatInvalid => e
       self.log_error self.exception_message(e)
     rescue => e # Error occurred processing job
-      self.log_error self.exception_message(e)
+      self.log_error self.exception_message(e) unless e.is_a?(Backburner::Job::RetryJob)
 
       unless job
         self.log_error "Error occurred before we were able to assign a job. Giving up without retrying!"
@@ -149,9 +150,11 @@ module Backburner
       # NB: There's a slight chance here that the connection to beanstalkd has
       # gone down between the time we reserved / processed the job and here.
       num_retries = job.stats.releases
-      retry_status = "failed: attempt #{num_retries+1} of #{queue_config.max_job_retries+1}"
-      if num_retries < queue_config.max_job_retries # retry again
-        delay = queue_config.retry_delay_proc.call(queue_config.retry_delay, num_retries) rescue queue_config.retry_delay
+      max_job_retries = resolve_max_job_retries(job.job_class)
+      retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
+      if num_retries < max_job_retries # retry again
+        retry_delay = resolve_retry_delay(job.job_class)
+        delay = resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries) rescue retry_delay
         job.retry(num_retries + 1, delay)
         self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
       else # retries failed, bury

data/lib/backburner/workers/threading.rb

@@ -3,9 +3,13 @@ require 'concurrent'
 module Backburner
   module Workers
     class Threading < Worker
+      attr_accessor :self_read, :self_write, :exit_on_shutdown
+
+      @shutdown_timeout = 10
+
       class << self
-        attr_accessor :shutdown
         attr_accessor :threads_number
+        attr_accessor :shutdown_timeout
       end
 
       # Custom initializer just to set @tubes_data
@@ -13,6 +17,7 @@ module Backburner
         @tubes_data = {}
         super
         self.process_tube_options
+        @exit_on_shutdown = true
       end
 
       # Used to prepare job queues before processing jobs.
@@ -48,16 +53,19 @@ module Backburner
             connection.on_reconnect = lambda { |conn| conn.tubes.watch!(tube_name) }
 
             # Make it work jobs using its own connection per thread
-            pool.post(connection) { |connection|
-              loop {
+            pool.post(connection) do |memo_connection|
+              # TODO: use read-write lock?
+              loop do
                 begin
-                  work_one_job(connection)
-
+                  break if @in_shutdown
+                  work_one_job(memo_connection)
                 rescue => e
                   log_error("Exception caught in thread pool loop. Continuing. -> #{e.message}\nBacktrace: #{e.backtrace}")
                 end
-              }
-            }
+              end
+
+              connection.close
+            end
           end
         end
 
@@ -111,18 +119,44 @@ module Backburner
 
       # Wait for the shutdown signel
       def wait_for_shutdown!
-        while !self.class.shutdown do
-          sleep 0.5
+        raise Interrupt while IO.select([self_read])
+      rescue Interrupt
+        shutdown
+      end
+
+      def shutdown_threadpools
+        @thread_pools.each { |_name, pool| pool.shutdown }
+        shutdown_time = Time.now
+        @in_shutdown = true
+        all_shutdown = @thread_pools.all? do |_name, pool|
+          time_to_wait = self.class.shutdown_timeout - (Time.now - shutdown_time).to_i
+          pool.wait_for_termination(time_to_wait) if time_to_wait > 0
         end
+      rescue Interrupt
+        log_info "graceful shutdown aborted, shutting down immediately"
+      ensure
+        kill unless all_shutdown
+      end
 
-        # Shutting down
-        # FIXME: Shut down each thread's connection
-        @thread_pools.each { |name, pool| pool.kill }
+      def kill
+        @thread_pools.each { |_name, pool| pool.kill unless pool.shutdown? }
       end
 
       def shutdown
-        Backburner::Workers::Threading.shutdown = true
-        super
+        log_info "beginning graceful worker shutdown"
+        shutdown_threadpools
+        super if @exit_on_shutdown
+      end
+
+      # Registers signal handlers TERM and INT to trigger
+      def register_signal_handlers!
+        @self_read, @self_write = IO.pipe
+        %w[TERM INT].each do |sig|
+          trap(sig) do
+            raise Interrupt if @in_shutdown
+            self_write.puts(sig)
+          end
+        end
       end
     end # Threading
   end # Workers

data/test/fixtures/test_jobs.rb

@@ -13,6 +13,24 @@ class TestJob
   def self.perform(x, y); $worker_test_count += x + y; end
 end
 
+class TestSlowJob
+  include Backburner::Queue
+  queue_priority :medium
+  queue_respond_timeout 300
+  def self.perform(x, y); sleep 1; $worker_test_count += x + y; end
+end
+
+class TestStuckJob
+  include Backburner::Queue
+  queue_priority :medium
+  queue_respond_timeout 300
+  def self.perform(_x, _y)
+    loop do
+      sleep 0.5
+    end
+  end
+end
+
 class TestFailJob
   include Backburner::Queue
   def self.perform(x, y); raise RuntimeError; end
@@ -36,6 +54,27 @@ class TestConfigurableRetryJob
   end
 end
 
+class TestRetryWithQueueOverridesJob
+  include Backburner::Queue
+  def self.perform(retry_count)
+    $worker_test_count += 1
+    raise RuntimeError unless $worker_test_count > retry_count
+    $worker_success = true
+  end
+
+  def self.queue_max_job_retries
+    3
+  end
+
+  def self.queue_retry_delay
+    0
+  end
+
+  def self.queue_retry_delay_proc
+    lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 2) }
+  end
+end
+
 class TestAsyncJob
   include Backburner::Performable
   def self.foo(x, y); $worker_test_count = x * y; end

data/test/helpers_test.rb

@@ -179,4 +179,100 @@ describe "Backburner::Helpers module" do
       assert_equal 300, resolve_respond_timeout(nil)
     end
   end # resolve_respond_timeout
+
+  describe "for resolve_max_job_retries method" do
+    before do
+      @original_max_job_retries = Backburner.configuration.max_job_retries
+      Backburner.configure { |config| config.max_job_retries = 300 }
+    end
+    after { Backburner.configure { |config| config.max_job_retries = @original_max_job_retries } }
+
+    it "supports fix num max_job_retries" do
+      assert_equal 500, resolve_max_job_retries(500)
+    end
+
+    it "supports classes which respond to queue_max_job_retries" do
+      job = stub(:queue_max_job_retries => 600)
+      assert_equal 600, resolve_max_job_retries(job)
+    end
+
+    it "supports classes which return null queue_max_job_retries" do
+      job = stub(:queue_max_job_retries => nil)
+      assert_equal 300, resolve_max_job_retries(job)
+    end
+
+    it "supports classes which don't respond to queue_max_job_retries" do
+      job = stub(:fake => true)
+      assert_equal 300, resolve_max_job_retries(job)
+    end
+
+    it "supports default max_job_retries for null values" do
+      assert_equal 300, resolve_max_job_retries(nil)
+    end
+  end # resolve_max_job_retries
+
+  describe "for resolve_retry_delay method" do
+    before do
+      @original_retry_delay = Backburner.configuration.retry_delay
+      Backburner.configure { |config| config.retry_delay = 300 }
+    end
+    after { Backburner.configure { |config| config.retry_delay = @original_retry_delay } }
+
+    it "supports fix num retry_delay" do
+      assert_equal 500, resolve_retry_delay(500)
+    end
+
+    it "supports classes which respond to queue_retry_delay" do
+      job = stub(:queue_retry_delay => 600)
+      assert_equal 600, resolve_retry_delay(job)
+    end
+
+    it "supports classes which return null queue_retry_delay" do
+      job = stub(:queue_retry_delay => nil)
+      assert_equal 300, resolve_retry_delay(job)
+    end
+
+    it "supports classes which don't respond to queue_retry_delay" do
+      job = stub(:fake => true)
+      assert_equal 300, resolve_retry_delay(job)
+    end
+
+    it "supports default retry_delay for null values" do
+      assert_equal 300, resolve_retry_delay(nil)
+    end
+  end # resolve_retry_delay
+
+  describe "for resolve_retry_delay_proc method" do
+    before do
+      @config_retry_delay_proc = lambda { |x, y| x + y } # Default config proc adds two values
+      @override_delay_proc = lambda { |x, y| x - y } # Overriden proc subtracts values
+      @original_retry_delay_proc = Backburner.configuration.retry_delay_proc
+      Backburner.configure { |config| config.retry_delay_proc = @config_retry_delay_proc }
+    end
+    after { Backburner.configure { |config| config.retry_delay_proc = @original_retry_delay_proc } }
+
+    # Rather than compare Procs execute them and compare the output
+    it "supports proc retry_delay_proc" do
+      assert_equal @override_delay_proc.call(2, 1), resolve_retry_delay_proc(@override_delay_proc).call(2, 1)
+    end
+
+    it "supports classes which respond to queue_retry_delay_proc" do
+      job = stub(:queue_retry_delay_proc => @override_delay_proc)
+      assert_equal @override_delay_proc.call(2, 1), resolve_retry_delay_proc(job).call(2, 1)
+    end
+
+    it "supports classes which return null queue_retry_delay_proc" do
+      job = stub(:queue_retry_delay_proc => nil)
+      assert_equal @original_retry_delay_proc.call(2, 1), resolve_retry_delay_proc(job).call(2, 1)
+    end
+
+    it "supports classes which don't respond to queue_retry_delay_proc" do
+      job = stub(:fake => true)
+      assert_equal @original_retry_delay_proc.call(2, 1), resolve_retry_delay_proc(job).call(2, 1)
+    end
+
+    it "supports default retry_delay_proc for null values" do
+      assert_equal @original_retry_delay_proc.call(2, 1), resolve_retry_delay_proc(nil).call(2, 1)
+    end
+  end # resolve_retry_delay_proc
 end

data/test/job_test.rb

@@ -15,16 +15,29 @@ describe "Backburner::Job module" do
   describe "for initialize" do
     describe "with hash" do
       before do
-        @task_body =  { "class" => "NewsletterSender", "args" => ["foo@bar.com", "bar@foo.com"] }
-        @task = stub(:body => @task_body, :ttr => 120, :delete => true, :bury => true)
+        @task = stub(:body => task_body, :ttr => 120, :delete => true, :bury => true)
       end
 
-      it "should create job with correct task data" do
-        @job = Backburner::Job.new(@task)
-        assert_equal @task, @job.task
-        assert_equal ["class", "args"], @job.body.keys
-        assert_equal @task_body["class"], @job.name
-        assert_equal @task_body["args"], @job.args
+      describe "with string keys" do
+        let(:task_body) { { "class" => "NewsletterSender", "args" => ["foo@bar.com", "bar@foo.com"] } }
+        it "should create job with correct task data" do
+          @job = Backburner::Job.new(@task)
+          assert_equal @task, @job.task
+          assert_equal ["class", "args"], @job.body.keys
+          assert_equal task_body["class"], @job.name
+          assert_equal task_body["args"], @job.args
+        end
+      end
+
+      describe "with symbol keys" do
+        let(:task_body) { { :class => "NewsletterSender", :args => ["foo@bar.com", "bar@foo.com"] } }
+        it "should create job with correct task data" do
+          @job = Backburner::Job.new(@task)
+          assert_equal @task, @job.task
+          assert_equal [:class, :args], @job.body.keys
+          assert_equal task_body[:class], @job.name
+          assert_equal task_body[:args], @job.args
+        end
       end
     end # with hash
 
@@ -100,16 +113,41 @@ describe "Backburner::Job module" do
   end # process
 
   describe "for simple delegation method" do
-    before do
-      @task_body =  { "class" => "NestedDemo::TestJobC", "args" => [56] }
-      @task = stub(:body => @task_body, :ttr => 120, :delete => true, :bury => true)
-      @task.expects(:bury).once
+    describe "with valid class" do
+      before do
+        @task_body =  { "class" => "NestedDemo::TestJobC", "args" => [56] }
+        @task = stub(:body => @task_body, :ttr => 120, :delete => true, :bury => true)
+        @task.expects(:bury).once
+      end
+
+      it "should call bury for task" do
+        @job = Backburner::Job.new(@task)
+        @job.bury
+      end # bury
     end
 
-    it "should call bury for task" do
-      @job = Backburner::Job.new(@task)
-      @job.bury
-    end # bury
+    describe "with invalid class" do
+      before do
+        @task_body = { "class" => "AnUnknownClass", "args" => [] }
+        @task = stub(:body => @task_body, :ttr => 120, :delete => true, :bury => true, :release => true)
+      end
+
+      it "should call bury for task" do
+        @task.expects(:bury).once
+        @job = Backburner::Job.new(@task)
+        Backburner::Hooks.expects(:invoke_hook_events)
+          .with("AnUnknownClass", :on_bury, anything)
+        @job.bury
+      end
+
+      it "should call retry for task" do
+        @task.expects(:release).once
+        @job = Backburner::Job.new(@task)
+        Backburner::Hooks.expects(:invoke_hook_events)
+          .with("AnUnknownClass", :on_retry, 0, is_a(Integer), anything)
+        @job.retry(0, 0)
+      end
+    end
   end # simple delegation
 
   describe "timing out for various values of ttr" do

data/test/queue_test.rb

@@ -44,4 +44,26 @@ describe "Backburner::Queue module" do
       assert_equal 300, NestedDemo::TestJobB.queue_respond_timeout
     end
   end # queue_respond_timeout
+
+  describe "for queue_max_job_retries assignment method" do
+    it "should allow queue max_job_retries to be assigned" do
+      NestedDemo::TestJobB.queue_max_job_retries(5)
+      assert_equal 5, NestedDemo::TestJobB.queue_max_job_retries
+    end
+  end # queue_max_job_retries
+
+  describe "for queue_retry_delay assignment method" do
+    it "should allow queue retry_delay to be assigned" do
+      NestedDemo::TestJobB.queue_retry_delay(300)
+      assert_equal 300, NestedDemo::TestJobB.queue_retry_delay
+    end
+  end # queue_retry_delay
+
+  describe "for queue_retry_delay_proc assignment method" do
+    it "should allow queue retry_delay_proc to be assigned" do
+      retry_delay_proc = lambda { |x, y| x - y }
+      NestedDemo::TestJobB.queue_retry_delay_proc(retry_delay_proc)
+      assert_equal retry_delay_proc.call(2, 1), NestedDemo::TestJobB.queue_retry_delay_proc.call(2, 1)
+    end
+  end # queue_retry_delay_proc
 end # Backburner::Queue

data/test/test_helper.rb

@@ -6,7 +6,7 @@ begin
 rescue LoadError
   require 'mocha'
 end
-$:.unshift File.expand_path("../../lib")
+$LOAD_PATH.unshift File.expand_path("lib")
 require 'backburner'
 require File.expand_path('../helpers/templogger', __FILE__)
 
@@ -92,12 +92,12 @@ class MiniTest::Spec
   end
 
   # pop_one_job(tube_name)
-  def pop_one_job(tube_name=Backburner.configuration.primary_queue, &block)
+  def pop_one_job(tube_name=Backburner.configuration.primary_queue)
     tube_name  = [Backburner.configuration.tube_namespace, tube_name].join(".")
     connection = beanstalk_connection
     connection.tubes.watch!(tube_name)
     silenced(3) { @res = connection.tubes.reserve }
-    yield @res, JSON.parse(@res.body)
+    yield @res, Backburner.configuration.job_parser_proc.call(@res.body)
   ensure
     connection.close if connection
   end

data/test/worker_test.rb

@@ -125,4 +125,33 @@ describe "Backburner::Worker module" do
       assert_equal ['baz', 'bam'], worker.tube_names
     end
   end # tube_names
+
+  describe "for custom serialization" do
+    before do
+      Backburner.configure do |config|
+        @old_parser = config.job_parser_proc
+        @old_serializer = config.job_serializer_proc
+        config.job_parser_proc = lambda { |body| Marshal.load(body) }
+        config.job_serializer_proc = lambda { |body| Marshal.dump(body) }
+      end
+    end
+
+    after do
+      clear_jobs!('test-plain')
+      Backburner.configure do |config|
+        config.job_parser_proc = @old_parser
+        config.job_serializer_proc = @old_serializer
+      end
+    end
+
+    it "should support enqueuing a job" do
+      Backburner::Worker.enqueue TestPlainJob, [7, 9], :ttr => 100, :pri => 2000
+      pop_one_job("test-plain") do |job, body|
+        assert_equal "TestPlainJob", body[:class]
+        assert_equal [7, 9], body[:args]
+        assert_equal 100, job.ttr
+        assert_equal 2000, job.pri
+      end
+    end
+  end # custom serialization
 end # Backburner::Worker

data/test/workers/forking_worker_test.rb

@@ -11,7 +11,7 @@ describe "Backburner::Workers::Forking module" do
   describe "for prepare method" do
     it "should make tube names array always unique to avoid duplication" do
       worker = @worker_class.new(["foo", "demo.test.foo"])
-      worker.prepare
+      capture_stdout { worker.prepare }
       assert_equal ["demo.test.foo"], worker.tube_names
     end
 
@@ -96,6 +96,7 @@ describe "Backburner::Workers::Forking module" do
 
     after do
       @templogger.close
+      Backburner.configuration.logger = nil
       clear_jobs!('response')
       clear_jobs!('bar.foo.1', 'bar.foo.2', 'bar.foo.3', 'bar.foo.4', 'bar.foo.5')
     end

data/test/workers/simple_worker_test.rb

@@ -11,7 +11,7 @@ describe "Backburner::Workers::Simple module" do
   describe "for prepare method" do
     it "should make tube names array always unique to avoid duplication" do
       worker = @worker_class.new(["foo", "demo.test.foo"])
-      worker.prepare
+      capture_stdout { worker.prepare }
       assert_equal ["demo.test.foo"], worker.tube_names
     end
 
@@ -229,6 +229,33 @@ describe "Backburner::Workers::Simple module" do
       assert_equal true, $worker_success
     end
 
+    it "should allow queue override of retries" do
+      max_job_retries = TestRetryWithQueueOverridesJob.queue_max_job_retries
+      clear_jobs!('foo.bar')
+      Backburner.configure do |config|
+         # Config should be overridden by queue overrides
+        config.max_job_retries = 20
+        config.retry_delay = 60
+        #config.retry_delay_proc = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) } # default retry_delay_proc
+      end
+      @worker_class.enqueue TestRetryWithQueueOverridesJob, [max_job_retries], :queue => 'foo.bar'
+      out = []
+      (max_job_retries + 1).times do
+        out << silenced(5) do
+          worker = @worker_class.new('foo.bar')
+          worker.prepare
+          worker.work_one_job
+        end
+      end
+      assert_match(/attempt 1 of 4, retrying in 0/, out.first)
+      assert_match(/attempt 2 of 4, retrying in 1/, out[1])
+      assert_match(/attempt 3 of 4, retrying in 4/, out[2])
+      assert_match(/Completed TestRetryWithQueueOverridesJob/m, out.last)
+      refute_match(/failed/, out.last)
+      assert_equal 4, $worker_test_count
+      assert_equal true, $worker_success
+    end
+
     it "should support event hooks without retry" do
       $hooked_fail_count = 0
       clear_jobs!('foo.bar.events')
@@ -309,7 +336,7 @@ describe "Backburner::Workers::Simple module" do
       worker = @worker_class.new('foo.bar')
       connection = mock('connection')
       worker.expects(:reserve_job).with(connection).returns(stub_everything('job'))
-      worker.work_one_job(connection)
+      capture_stdout { worker.work_one_job(connection) }
     end
 
     after do

data/test/workers/threading_worker_test.rb

@@ -6,24 +6,25 @@ describe "Backburner::Workers::Threading worker" do
   before do
     Backburner.default_queues.clear
     @worker_class = Backburner::Workers::Threading
+    @worker_class.shutdown_timeout = 2
   end
 
   describe "for prepare method" do
     it "should make tube names array always unique to avoid duplication" do
       worker = @worker_class.new(["foo", "demo.test.foo"])
-      worker.prepare
+      capture_stdout { worker.prepare }
       assert_equal ["demo.test.foo"], worker.tube_names
     end
 
     it 'creates a thread pool per queue' do
       worker = @worker_class.new(%w(foo bar))
-      out = capture_stdout { worker.prepare }
+      capture_stdout { worker.prepare }
       assert_equal 2, worker.instance_variable_get("@thread_pools").keys.size
     end
 
     it 'uses Concurrent.processor_count if no custom thread count is provided' do
       worker = @worker_class.new("foo")
-      out = capture_stdout { worker.prepare }
+      capture_stdout { worker.prepare }
       assert_equal ::Concurrent.processor_count, worker.instance_variable_get("@thread_pools")["demo.test.foo"].max_length
     end
   end # prepare
@@ -54,15 +55,50 @@ describe "Backburner::Workers::Threading worker" do
     before do
       @worker = @worker_class.new(["foo:3"])
       capture_stdout { @worker.prepare }
+      $worker_test_count = 0
+      $worker_success = false
     end
 
     it 'runs work_on_job per thread' do
       clear_jobs!("foo")
       job_count=10
-      job_count.times { @worker_class.enqueue TestJob, [1, 0], :queue => "foo" } # TestJob adds the given arguments together and then to $worker_test_count
-      @worker.start(false) # don't wait for shutdown
-      sleep 0.5 # Wait for threads to do their work
+      # TestJob adds the given arguments together and then to $worker_test_count
+      job_count.times { @worker_class.enqueue TestJob, [1, 0], :queue => "foo" }
+      capture_stdout do
+        @worker.start(false) # don't wait for shutdown
+        sleep 0.5 # Wait for threads to do their work
+      end
       assert_equal job_count, $worker_test_count
     end
   end # working a queue
+
+  describe 'shutting down' do
+    before do
+      @thread_count = 3
+      @worker = @worker_class.new(["threaded-shutdown:#{@thread_count}"])
+      @worker.exit_on_shutdown = false
+      $worker_test_count = 0
+      clear_jobs!("threaded-shutdown")
+    end
+
+    it 'gracefully exits and completes all in-flight jobs' do
+      6.times { @worker_class.enqueue TestSlowJob, [1, 0], :queue => "threaded-shutdown" }
+      Thread.new { sleep 0.1; @worker.self_write.puts("TERM") }
+      capture_stdout do
+        @worker.start
+      end
+
+      assert_equal @thread_count, $worker_test_count
+    end
+
+    it 'forces an exit when a job is stuck' do
+      6.times { @worker_class.enqueue TestStuckJob, [1, 0], :queue => "threaded-shutdown" }
+      Thread.new { sleep 0.1; @worker.self_write.puts("TERM") }
+      capture_stdout do
+        @worker.start
+      end
+
+      assert_equal 0, $worker_test_count
+    end
+  end
 end

data/test/workers/threads_on_fork_worker_test.rb

@@ -70,7 +70,7 @@ describe "Backburner::Workers::ThreadsOnFork module" do
 
     it "should make tube names array always unique to avoid duplication" do
       worker = @worker_class.new(["foo", "demo.test.foo"])
-      worker.prepare
+      capture_stdout { worker.prepare }
       assert_equal ["demo.test.foo"], worker.tube_names
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: backburner
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.6.0
 platform: ruby
 authors:
 - Nathan Esquenazi
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-02 00:00:00.000000000 Z
+date: 2021-12-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: beaneater
@@ -43,6 +43,9 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.1
   type: :runtime
@@ -50,6 +53,9 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.1
 - !ruby/object:Gem::Dependency
@@ -94,20 +100,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Beanstalk background job processing made easy
 email:
 - nesquena@gmail.com
@@ -179,7 +171,7 @@ homepage: http://github.com/nesquena/backburner
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -194,9 +186,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.8
-signing_key: 
+rubygems_version: 3.0.8
+signing_key:
 specification_version: 4
 summary: Reliable beanstalk background job processing made easy for Ruby and Sinatra
 test_files: