backburner 1.5.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 9dfd7f06d2b08cb31ed2cf463506169544e0920d
-  data.tar.gz: 5d7585331fa5ccc082901017e9149d9982cb9de3
+SHA256:
+  metadata.gz: 93c9b77199209b894ae963d9474a3234212225021537f99b9fee11433e73f6c4
+  data.tar.gz: 49e775e7a32ccc85a557f27ba89a146617553b45e0a252ddfe12f84d1e501706
 SHA512:
-  metadata.gz: 38f1deb4da6faef87915b8fca99d99209adf76c5059f3478a75883b38a5e7abf89a9c0c9cec439641530f2b3e272362bfb0b8cc3343b9c1f9fcabf256b5bee24
-  data.tar.gz: d881653ea35fa55df59d53c5e104bc6dea13d857cabebacd8ecb5ba8ff845b9e736263ac7d1c386bbd6ff1d062e9abe0137ff8cf995262accd7231c7e4dc5a0b
+  metadata.gz: f6e5cdea7a5e3092b8dfc61199ccabf1eb898b3eae4c5645171a7fecc415d7c1c467d1fd173d78732dee6c60bfd2f5928cb728a52e13269bbd951d6d304a2ceb
+  data.tar.gz: e0da0a58b32ea88704e33a4e19a03b3f3b1c8cb63428091743953112841e6b2fc1bf27d5de18d51a8777cad9ee45e069a02a0f9cc132fdf8a0e036bd0379e162

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## Version 1.6.0 (December 30 2021)
+
+* TBD (please help backfill)
+
 ## Version 1.5.0 (September 10 2018)
 
 * TBD

data/README.md

@@ -1,6 +1,6 @@
 # Backburner [![Build Status](https://travis-ci.org/nesquena/backburner.svg?branch=master)](https://travis-ci.org/nesquena/backburner)
 
-Backburner is a [beanstalkd](http://kr.github.com/beanstalkd/)-powered job queue that can handle a very high volume of jobs.
+Backburner is a [beanstalkd](https://beanstalkd.github.io/)-powered job queue that can handle a very high volume of jobs.
 You create background jobs and place them on multiple work queues to be processed later.
 
 Processing background jobs reliably has never been easier than with beanstalkd and Backburner. This gem works with any ruby-based
@@ -65,7 +65,7 @@ In the end, **beanstalk is the ideal job queue** while also being ridiculously e
 
 ## Installation
 
-First, you probably want to [install beanstalkd](http://kr.github.com/beanstalkd/download.html), which powers the job queues.
+First, you probably want to [install beanstalkd](https://beanstalkd.github.io/download.html), which powers the job queues.
 Depending on your platform, this should be as simple as (for Ubuntu):
 
     $ sudo apt-get install beanstalkd
@@ -163,10 +163,25 @@ class NewsletterJob
     1000 # most urgent priority is 0
   end
 
-  # optional, defaults to respond_timeout
+  # optional, defaults to respond_timeout in config
   def self.queue_respond_timeout
     300 # number of seconds before job times out, 0 to avoid timeout. NB: A timeout of 1 second will likely lead to race conditions between Backburner and beanstalkd and should be avoided
   end
+
+  # optional, defaults to retry_delay_proc in config
+  def self.queue_retry_delay_proc
+    lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 5) }
+  end
+
+  # optional, defaults to retry_delay in config
+  def self.queue_retry_delay
+    5
+  end
+
+  # optional, defaults to max_job_retries in config
+  def self.queue_max_job_retries
+    5
+  end
 end
 ```
 

data/lib/backburner/helpers.rb

@@ -138,5 +138,56 @@ module Backburner
       end
     end
 
+    # Resolves max retries based on the value given. Can be integer, a class or nothing
+    #
+    # @example
+    #  resolve_max_job_retries(5) => 5
+    #  resolve_max_job_retries(FooBar) => <queue max_job_retries>
+    #  resolve_max_job_retries(nil) => <default max_job_retries>
+    #
+    def resolve_max_job_retries(retries)
+      if retries.respond_to?(:queue_max_job_retries)
+        resolve_max_job_retries(retries.queue_max_job_retries)
+      elsif retries.is_a?(Integer) # numerical
+        retries
+      else # default
+        Backburner.configuration.max_job_retries
+      end
+    end
+
+    # Resolves retry delay based on the value given. Can be integer, a class or nothing
+    #
+    # @example
+    #  resolve_retry_delay(5) => 5
+    #  resolve_retry_delay(FooBar) => <queue retry_delay>
+    #  resolve_retry_delay(nil) => <default retry_delay>
+    #
+    def resolve_retry_delay(delay)
+      if delay.respond_to?(:queue_retry_delay)
+        resolve_retry_delay(delay.queue_retry_delay)
+      elsif delay.is_a?(Integer) # numerical
+        delay
+      else # default
+        Backburner.configuration.retry_delay
+      end
+    end
+
+    # Resolves retry delay proc based on the value given. Can be proc, a class or nothing
+    #
+    # @example
+    #  resolve_retry_delay_proc(proc) => proc
+    #  resolve_retry_delay_proc(FooBar) => <queue retry_delay_proc>
+    #  resolve_retry_delay_proc(nil) => <default retry_delay_proc>
+    #
+    def resolve_retry_delay_proc(proc)
+      if proc.respond_to?(:queue_retry_delay_proc)
+        resolve_retry_delay_proc(proc.queue_retry_delay_proc)
+      elsif proc.is_a?(Proc)
+        proc
+      else # default
+        Backburner.configuration.retry_delay_proc
+      end
+    end
+
   end # Helpers
 end # Backburner

data/lib/backburner/job.rb

@@ -73,8 +73,6 @@ module Backburner
       task.release(delay: delay)
     end
 
-    protected
-
     # Returns the class for the job handler
     #
     # @example
@@ -86,6 +84,8 @@ module Backburner
       handler
     end
 
+    protected
+
     # Attempts to return a constantized job name, otherwise reverts to the name string
     #
     # @example

data/lib/backburner/queue.rb

@@ -4,6 +4,9 @@ module Backburner
       base.instance_variable_set(:@queue_name, nil)
       base.instance_variable_set(:@queue_priority, nil)
       base.instance_variable_set(:@queue_respond_timeout, nil)
+      base.instance_variable_set(:@queue_max_job_retries, nil)
+      base.instance_variable_set(:@queue_retry_delay, nil)
+      base.instance_variable_set(:@queue_retry_delay_proc, nil)
       base.instance_variable_set(:@queue_jobs_limit, nil)
       base.instance_variable_set(:@queue_garbage_limit, nil)
       base.instance_variable_set(:@queue_retry_limit, nil)
@@ -54,6 +57,48 @@ module Backburner
         end
       end
 
+      # Returns or assigns queue max_job_retries for this job
+      #
+      # @example
+      #   queue_max_job_retries 120
+      #   @klass.queue_max_job_retries # => 120
+      #
+      def queue_max_job_retries(delay=nil)
+        if delay
+          @queue_max_job_retries = delay
+        else # accessor
+          @queue_max_job_retries
+        end
+      end
+
+      # Returns or assigns queue retry_delay for this job
+      #
+      # @example
+      #   queue_retry_delay 120
+      #   @klass.queue_retry_delay # => 120
+      #
+      def queue_retry_delay(delay=nil)
+        if delay
+          @queue_retry_delay = delay
+        else # accessor
+          @queue_retry_delay
+        end
+      end
+
+      # Returns or assigns queue retry_delay_proc for this job
+      #
+      # @example
+      #   queue_retry_delay_proc lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 2) }
+      #   @klass.queue_retry_delay_proc # => lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 2) }
+      #
+      def queue_retry_delay_proc(proc=nil)
+        if proc
+          @queue_retry_delay_proc = proc
+        else # accessor
+          @queue_retry_delay_proc
+        end
+      end
+
       # Returns or assigns queue parallel active jobs limit (only ThreadsOnFork and Threading workers)
       #
       # @example

data/lib/backburner/version.rb

@@ -1,3 +1,3 @@
 module Backburner
-  VERSION = "1.5.0"
+  VERSION = "1.6.0"
 end

data/lib/backburner/worker.rb

@@ -150,9 +150,11 @@ module Backburner
       # NB: There's a slight chance here that the connection to beanstalkd has
       # gone down between the time we reserved / processed the job and here.
       num_retries = job.stats.releases
-      retry_status = "failed: attempt #{num_retries+1} of #{queue_config.max_job_retries+1}"
-      if num_retries < queue_config.max_job_retries # retry again
-        delay = queue_config.retry_delay_proc.call(queue_config.retry_delay, num_retries) rescue queue_config.retry_delay
+      max_job_retries = resolve_max_job_retries(job.job_class)
+      retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
+      if num_retries < max_job_retries # retry again
+        retry_delay = resolve_retry_delay(job.job_class)
+        delay = resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries) rescue retry_delay
         job.retry(num_retries + 1, delay)
         self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
       else # retries failed, bury

data/test/fixtures/test_jobs.rb

@@ -54,6 +54,27 @@ class TestConfigurableRetryJob
   end
 end
 
+class TestRetryWithQueueOverridesJob
+  include Backburner::Queue
+  def self.perform(retry_count)
+    $worker_test_count += 1
+    raise RuntimeError unless $worker_test_count > retry_count
+    $worker_success = true
+  end
+
+  def self.queue_max_job_retries
+    3
+  end
+
+  def self.queue_retry_delay
+    0
+  end
+
+  def self.queue_retry_delay_proc
+    lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 2) }
+  end
+end
+
 class TestAsyncJob
   include Backburner::Performable
   def self.foo(x, y); $worker_test_count = x * y; end

data/test/helpers_test.rb

@@ -179,4 +179,100 @@ describe "Backburner::Helpers module" do
       assert_equal 300, resolve_respond_timeout(nil)
     end
   end # resolve_respond_timeout
+
+  describe "for resolve_max_job_retries method" do
+    before do
+      @original_max_job_retries = Backburner.configuration.max_job_retries
+      Backburner.configure { |config| config.max_job_retries = 300 }
+    end
+    after { Backburner.configure { |config| config.max_job_retries = @original_max_job_retries } }
+
+    it "supports fix num max_job_retries" do
+      assert_equal 500, resolve_max_job_retries(500)
+    end
+
+    it "supports classes which respond to queue_max_job_retries" do
+      job = stub(:queue_max_job_retries => 600)
+      assert_equal 600, resolve_max_job_retries(job)
+    end
+
+    it "supports classes which return null queue_max_job_retries" do
+      job = stub(:queue_max_job_retries => nil)
+      assert_equal 300, resolve_max_job_retries(job)
+    end
+
+    it "supports classes which don't respond to queue_max_job_retries" do
+      job = stub(:fake => true)
+      assert_equal 300, resolve_max_job_retries(job)
+    end
+
+    it "supports default max_job_retries for null values" do
+      assert_equal 300, resolve_max_job_retries(nil)
+    end
+  end # resolve_max_job_retries
+
+  describe "for resolve_retry_delay method" do
+    before do
+      @original_retry_delay = Backburner.configuration.retry_delay
+      Backburner.configure { |config| config.retry_delay = 300 }
+    end
+    after { Backburner.configure { |config| config.retry_delay = @original_retry_delay } }
+
+    it "supports fix num retry_delay" do
+      assert_equal 500, resolve_retry_delay(500)
+    end
+
+    it "supports classes which respond to queue_retry_delay" do
+      job = stub(:queue_retry_delay => 600)
+      assert_equal 600, resolve_retry_delay(job)
+    end
+
+    it "supports classes which return null queue_retry_delay" do
+      job = stub(:queue_retry_delay => nil)
+      assert_equal 300, resolve_retry_delay(job)
+    end
+
+    it "supports classes which don't respond to queue_retry_delay" do
+      job = stub(:fake => true)
+      assert_equal 300, resolve_retry_delay(job)
+    end
+
+    it "supports default retry_delay for null values" do
+      assert_equal 300, resolve_retry_delay(nil)
+    end
+  end # resolve_retry_delay
+
+  describe "for resolve_retry_delay_proc method" do
+    before do
+      @config_retry_delay_proc = lambda { |x, y| x + y } # Default config proc adds two values
+      @override_delay_proc = lambda { |x, y| x - y } # Overriden proc subtracts values
+      @original_retry_delay_proc = Backburner.configuration.retry_delay_proc
+      Backburner.configure { |config| config.retry_delay_proc = @config_retry_delay_proc }
+    end
+    after { Backburner.configure { |config| config.retry_delay_proc = @original_retry_delay_proc } }
+
+    # Rather than compare Procs execute them and compare the output
+    it "supports proc retry_delay_proc" do
+      assert_equal @override_delay_proc.call(2, 1), resolve_retry_delay_proc(@override_delay_proc).call(2, 1)
+    end
+
+    it "supports classes which respond to queue_retry_delay_proc" do
+      job = stub(:queue_retry_delay_proc => @override_delay_proc)
+      assert_equal @override_delay_proc.call(2, 1), resolve_retry_delay_proc(job).call(2, 1)
+    end
+
+    it "supports classes which return null queue_retry_delay_proc" do
+      job = stub(:queue_retry_delay_proc => nil)
+      assert_equal @original_retry_delay_proc.call(2, 1), resolve_retry_delay_proc(job).call(2, 1)
+    end
+
+    it "supports classes which don't respond to queue_retry_delay_proc" do
+      job = stub(:fake => true)
+      assert_equal @original_retry_delay_proc.call(2, 1), resolve_retry_delay_proc(job).call(2, 1)
+    end
+
+    it "supports default retry_delay_proc for null values" do
+      assert_equal @original_retry_delay_proc.call(2, 1), resolve_retry_delay_proc(nil).call(2, 1)
+    end
+  end # resolve_retry_delay_proc
 end

data/test/queue_test.rb

@@ -44,4 +44,26 @@ describe "Backburner::Queue module" do
       assert_equal 300, NestedDemo::TestJobB.queue_respond_timeout
     end
   end # queue_respond_timeout
+
+  describe "for queue_max_job_retries assignment method" do
+    it "should allow queue max_job_retries to be assigned" do
+      NestedDemo::TestJobB.queue_max_job_retries(5)
+      assert_equal 5, NestedDemo::TestJobB.queue_max_job_retries
+    end
+  end # queue_max_job_retries
+
+  describe "for queue_retry_delay assignment method" do
+    it "should allow queue retry_delay to be assigned" do
+      NestedDemo::TestJobB.queue_retry_delay(300)
+      assert_equal 300, NestedDemo::TestJobB.queue_retry_delay
+    end
+  end # queue_retry_delay
+
+  describe "for queue_retry_delay_proc assignment method" do
+    it "should allow queue retry_delay_proc to be assigned" do
+      retry_delay_proc = lambda { |x, y| x - y }
+      NestedDemo::TestJobB.queue_retry_delay_proc(retry_delay_proc)
+      assert_equal retry_delay_proc.call(2, 1), NestedDemo::TestJobB.queue_retry_delay_proc.call(2, 1)
+    end
+  end # queue_retry_delay_proc
 end # Backburner::Queue

data/test/workers/simple_worker_test.rb

@@ -229,6 +229,33 @@ describe "Backburner::Workers::Simple module" do
       assert_equal true, $worker_success
     end
 
+    it "should allow queue override of retries" do
+      max_job_retries = TestRetryWithQueueOverridesJob.queue_max_job_retries
+      clear_jobs!('foo.bar')
+      Backburner.configure do |config|
+         # Config should be overridden by queue overrides
+        config.max_job_retries = 20
+        config.retry_delay = 60
+        #config.retry_delay_proc = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) } # default retry_delay_proc
+      end
+      @worker_class.enqueue TestRetryWithQueueOverridesJob, [max_job_retries], :queue => 'foo.bar'
+      out = []
+      (max_job_retries + 1).times do
+        out << silenced(5) do
+          worker = @worker_class.new('foo.bar')
+          worker.prepare
+          worker.work_one_job
+        end
+      end
+      assert_match(/attempt 1 of 4, retrying in 0/, out.first)
+      assert_match(/attempt 2 of 4, retrying in 1/, out[1])
+      assert_match(/attempt 3 of 4, retrying in 4/, out[2])
+      assert_match(/Completed TestRetryWithQueueOverridesJob/m, out.last)
+      refute_match(/failed/, out.last)
+      assert_equal 4, $worker_test_count
+      assert_equal true, $worker_success
+    end
+
     it "should support event hooks without retry" do
       $hooked_fail_count = 0
       clear_jobs!('foo.bar.events')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: backburner
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.6.0
 platform: ruby
 authors:
 - Nathan Esquenazi
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-09-19 00:00:00.000000000 Z
+date: 2021-12-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: beaneater
@@ -171,7 +171,7 @@ homepage: http://github.com/nesquena/backburner
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -186,9 +186,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.7
-signing_key: 
+rubygems_version: 3.0.8
+signing_key:
 specification_version: 4
 summary: Reliable beanstalk background job processing made easy for Ruby and Sinatra
 test_files: