backburner 1.5.0 → 1.6.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 9dfd7f06d2b08cb31ed2cf463506169544e0920d
4
- data.tar.gz: 5d7585331fa5ccc082901017e9149d9982cb9de3
2
+ SHA256:
3
+ metadata.gz: 93c9b77199209b894ae963d9474a3234212225021537f99b9fee11433e73f6c4
4
+ data.tar.gz: 49e775e7a32ccc85a557f27ba89a146617553b45e0a252ddfe12f84d1e501706
5
5
  SHA512:
6
- metadata.gz: 38f1deb4da6faef87915b8fca99d99209adf76c5059f3478a75883b38a5e7abf89a9c0c9cec439641530f2b3e272362bfb0b8cc3343b9c1f9fcabf256b5bee24
7
- data.tar.gz: d881653ea35fa55df59d53c5e104bc6dea13d857cabebacd8ecb5ba8ff845b9e736263ac7d1c386bbd6ff1d062e9abe0137ff8cf995262accd7231c7e4dc5a0b
6
+ metadata.gz: f6e5cdea7a5e3092b8dfc61199ccabf1eb898b3eae4c5645171a7fecc415d7c1c467d1fd173d78732dee6c60bfd2f5928cb728a52e13269bbd951d6d304a2ceb
7
+ data.tar.gz: e0da0a58b32ea88704e33a4e19a03b3f3b1c8cb63428091743953112841e6b2fc1bf27d5de18d51a8777cad9ee45e069a02a0f9cc132fdf8a0e036bd0379e162
data/CHANGELOG.md CHANGED
@@ -1,5 +1,9 @@
1
1
  # CHANGELOG
2
2
 
3
+ ## Version 1.6.0 (December 30 2021)
4
+
5
+ * TBD (please help backfill)
6
+
3
7
  ## Version 1.5.0 (September 10 2018)
4
8
 
5
9
  * TBD
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # Backburner [![Build Status](https://travis-ci.org/nesquena/backburner.svg?branch=master)](https://travis-ci.org/nesquena/backburner)
2
2
 
3
- Backburner is a [beanstalkd](http://kr.github.com/beanstalkd/)-powered job queue that can handle a very high volume of jobs.
3
+ Backburner is a [beanstalkd](https://beanstalkd.github.io/)-powered job queue that can handle a very high volume of jobs.
4
4
  You create background jobs and place them on multiple work queues to be processed later.
5
5
 
6
6
  Processing background jobs reliably has never been easier than with beanstalkd and Backburner. This gem works with any ruby-based
@@ -65,7 +65,7 @@ In the end, **beanstalk is the ideal job queue** while also being ridiculously e
65
65
 
66
66
  ## Installation
67
67
 
68
- First, you probably want to [install beanstalkd](http://kr.github.com/beanstalkd/download.html), which powers the job queues.
68
+ First, you probably want to [install beanstalkd](https://beanstalkd.github.io/download.html), which powers the job queues.
69
69
  Depending on your platform, this should be as simple as (for Ubuntu):
70
70
 
71
71
  $ sudo apt-get install beanstalkd
@@ -163,10 +163,25 @@ class NewsletterJob
163
163
  1000 # most urgent priority is 0
164
164
  end
165
165
 
166
- # optional, defaults to respond_timeout
166
+ # optional, defaults to respond_timeout in config
167
167
  def self.queue_respond_timeout
168
168
  300 # number of seconds before job times out, 0 to avoid timeout. NB: A timeout of 1 second will likely lead to race conditions between Backburner and beanstalkd and should be avoided
169
169
  end
170
+
171
+ # optional, defaults to retry_delay_proc in config
172
+ def self.queue_retry_delay_proc
173
+ lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 5) }
174
+ end
175
+
176
+ # optional, defaults to retry_delay in config
177
+ def self.queue_retry_delay
178
+ 5
179
+ end
180
+
181
+ # optional, defaults to max_job_retries in config
182
+ def self.queue_max_job_retries
183
+ 5
184
+ end
170
185
  end
171
186
  ```
172
187
 
@@ -138,5 +138,56 @@ module Backburner
138
138
  end
139
139
  end
140
140
 
141
+ # Resolves max retries based on the value given. Can be integer, a class or nothing
142
+ #
143
+ # @example
144
+ # resolve_max_job_retries(5) => 5
145
+ # resolve_max_job_retries(FooBar) => <queue max_job_retries>
146
+ # resolve_max_job_retries(nil) => <default max_job_retries>
147
+ #
148
+ def resolve_max_job_retries(retries)
149
+ if retries.respond_to?(:queue_max_job_retries)
150
+ resolve_max_job_retries(retries.queue_max_job_retries)
151
+ elsif retries.is_a?(Integer) # numerical
152
+ retries
153
+ else # default
154
+ Backburner.configuration.max_job_retries
155
+ end
156
+ end
157
+
158
+ # Resolves retry delay based on the value given. Can be integer, a class or nothing
159
+ #
160
+ # @example
161
+ # resolve_retry_delay(5) => 5
162
+ # resolve_retry_delay(FooBar) => <queue retry_delay>
163
+ # resolve_retry_delay(nil) => <default retry_delay>
164
+ #
165
+ def resolve_retry_delay(delay)
166
+ if delay.respond_to?(:queue_retry_delay)
167
+ resolve_retry_delay(delay.queue_retry_delay)
168
+ elsif delay.is_a?(Integer) # numerical
169
+ delay
170
+ else # default
171
+ Backburner.configuration.retry_delay
172
+ end
173
+ end
174
+
175
+ # Resolves retry delay proc based on the value given. Can be proc, a class or nothing
176
+ #
177
+ # @example
178
+ # resolve_retry_delay_proc(proc) => proc
179
+ # resolve_retry_delay_proc(FooBar) => <queue retry_delay_proc>
180
+ # resolve_retry_delay_proc(nil) => <default retry_delay_proc>
181
+ #
182
+ def resolve_retry_delay_proc(proc)
183
+ if proc.respond_to?(:queue_retry_delay_proc)
184
+ resolve_retry_delay_proc(proc.queue_retry_delay_proc)
185
+ elsif proc.is_a?(Proc)
186
+ proc
187
+ else # default
188
+ Backburner.configuration.retry_delay_proc
189
+ end
190
+ end
191
+
141
192
  end # Helpers
142
193
  end # Backburner
@@ -73,8 +73,6 @@ module Backburner
73
73
  task.release(delay: delay)
74
74
  end
75
75
 
76
- protected
77
-
78
76
  # Returns the class for the job handler
79
77
  #
80
78
  # @example
@@ -86,6 +84,8 @@ module Backburner
86
84
  handler
87
85
  end
88
86
 
87
+ protected
88
+
89
89
  # Attempts to return a constantized job name, otherwise reverts to the name string
90
90
  #
91
91
  # @example
@@ -4,6 +4,9 @@ module Backburner
4
4
  base.instance_variable_set(:@queue_name, nil)
5
5
  base.instance_variable_set(:@queue_priority, nil)
6
6
  base.instance_variable_set(:@queue_respond_timeout, nil)
7
+ base.instance_variable_set(:@queue_max_job_retries, nil)
8
+ base.instance_variable_set(:@queue_retry_delay, nil)
9
+ base.instance_variable_set(:@queue_retry_delay_proc, nil)
7
10
  base.instance_variable_set(:@queue_jobs_limit, nil)
8
11
  base.instance_variable_set(:@queue_garbage_limit, nil)
9
12
  base.instance_variable_set(:@queue_retry_limit, nil)
@@ -54,6 +57,48 @@ module Backburner
54
57
  end
55
58
  end
56
59
 
60
+ # Returns or assigns queue max_job_retries for this job
61
+ #
62
+ # @example
63
+ # queue_max_job_retries 120
64
+ # @klass.queue_max_job_retries # => 120
65
+ #
66
+ def queue_max_job_retries(delay=nil)
67
+ if delay
68
+ @queue_max_job_retries = delay
69
+ else # accessor
70
+ @queue_max_job_retries
71
+ end
72
+ end
73
+
74
+ # Returns or assigns queue retry_delay for this job
75
+ #
76
+ # @example
77
+ # queue_retry_delay 120
78
+ # @klass.queue_retry_delay # => 120
79
+ #
80
+ def queue_retry_delay(delay=nil)
81
+ if delay
82
+ @queue_retry_delay = delay
83
+ else # accessor
84
+ @queue_retry_delay
85
+ end
86
+ end
87
+
88
+ # Returns or assigns queue retry_delay_proc for this job
89
+ #
90
+ # @example
91
+ # queue_retry_delay_proc lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 2) }
92
+ # @klass.queue_retry_delay_proc # => lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 2) }
93
+ #
94
+ def queue_retry_delay_proc(proc=nil)
95
+ if proc
96
+ @queue_retry_delay_proc = proc
97
+ else # accessor
98
+ @queue_retry_delay_proc
99
+ end
100
+ end
101
+
57
102
  # Returns or assigns queue parallel active jobs limit (only ThreadsOnFork and Threading workers)
58
103
  #
59
104
  # @example
@@ -1,3 +1,3 @@
1
1
  module Backburner
2
- VERSION = "1.5.0"
2
+ VERSION = "1.6.0"
3
3
  end
@@ -150,9 +150,11 @@ module Backburner
150
150
  # NB: There's a slight chance here that the connection to beanstalkd has
151
151
  # gone down between the time we reserved / processed the job and here.
152
152
  num_retries = job.stats.releases
153
- retry_status = "failed: attempt #{num_retries+1} of #{queue_config.max_job_retries+1}"
154
- if num_retries < queue_config.max_job_retries # retry again
155
- delay = queue_config.retry_delay_proc.call(queue_config.retry_delay, num_retries) rescue queue_config.retry_delay
153
+ max_job_retries = resolve_max_job_retries(job.job_class)
154
+ retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
155
+ if num_retries < max_job_retries # retry again
156
+ retry_delay = resolve_retry_delay(job.job_class)
157
+ delay = resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries) rescue retry_delay
156
158
  job.retry(num_retries + 1, delay)
157
159
  self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
158
160
  else # retries failed, bury
@@ -54,6 +54,27 @@ class TestConfigurableRetryJob
54
54
  end
55
55
  end
56
56
 
57
+ class TestRetryWithQueueOverridesJob
58
+ include Backburner::Queue
59
+ def self.perform(retry_count)
60
+ $worker_test_count += 1
61
+ raise RuntimeError unless $worker_test_count > retry_count
62
+ $worker_success = true
63
+ end
64
+
65
+ def self.queue_max_job_retries
66
+ 3
67
+ end
68
+
69
+ def self.queue_retry_delay
70
+ 0
71
+ end
72
+
73
+ def self.queue_retry_delay_proc
74
+ lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 2) }
75
+ end
76
+ end
77
+
57
78
  class TestAsyncJob
58
79
  include Backburner::Performable
59
80
  def self.foo(x, y); $worker_test_count = x * y; end
data/test/helpers_test.rb CHANGED
@@ -179,4 +179,100 @@ describe "Backburner::Helpers module" do
179
179
  assert_equal 300, resolve_respond_timeout(nil)
180
180
  end
181
181
  end # resolve_respond_timeout
182
+
183
+ describe "for resolve_max_job_retries method" do
184
+ before do
185
+ @original_max_job_retries = Backburner.configuration.max_job_retries
186
+ Backburner.configure { |config| config.max_job_retries = 300 }
187
+ end
188
+ after { Backburner.configure { |config| config.max_job_retries = @original_max_job_retries } }
189
+
190
+ it "supports fix num max_job_retries" do
191
+ assert_equal 500, resolve_max_job_retries(500)
192
+ end
193
+
194
+ it "supports classes which respond to queue_max_job_retries" do
195
+ job = stub(:queue_max_job_retries => 600)
196
+ assert_equal 600, resolve_max_job_retries(job)
197
+ end
198
+
199
+ it "supports classes which return null queue_max_job_retries" do
200
+ job = stub(:queue_max_job_retries => nil)
201
+ assert_equal 300, resolve_max_job_retries(job)
202
+ end
203
+
204
+ it "supports classes which don't respond to queue_max_job_retries" do
205
+ job = stub(:fake => true)
206
+ assert_equal 300, resolve_max_job_retries(job)
207
+ end
208
+
209
+ it "supports default max_job_retries for null values" do
210
+ assert_equal 300, resolve_max_job_retries(nil)
211
+ end
212
+ end # resolve_max_job_retries
213
+
214
+ describe "for resolve_retry_delay method" do
215
+ before do
216
+ @original_retry_delay = Backburner.configuration.retry_delay
217
+ Backburner.configure { |config| config.retry_delay = 300 }
218
+ end
219
+ after { Backburner.configure { |config| config.retry_delay = @original_retry_delay } }
220
+
221
+ it "supports fix num retry_delay" do
222
+ assert_equal 500, resolve_retry_delay(500)
223
+ end
224
+
225
+ it "supports classes which respond to queue_retry_delay" do
226
+ job = stub(:queue_retry_delay => 600)
227
+ assert_equal 600, resolve_retry_delay(job)
228
+ end
229
+
230
+ it "supports classes which return null queue_retry_delay" do
231
+ job = stub(:queue_retry_delay => nil)
232
+ assert_equal 300, resolve_retry_delay(job)
233
+ end
234
+
235
+ it "supports classes which don't respond to queue_retry_delay" do
236
+ job = stub(:fake => true)
237
+ assert_equal 300, resolve_retry_delay(job)
238
+ end
239
+
240
+ it "supports default retry_delay for null values" do
241
+ assert_equal 300, resolve_retry_delay(nil)
242
+ end
243
+ end # resolve_retry_delay
244
+
245
+ describe "for resolve_retry_delay_proc method" do
246
+ before do
247
+ @config_retry_delay_proc = lambda { |x, y| x + y } # Default config proc adds two values
248
+ @override_delay_proc = lambda { |x, y| x - y } # Overriden proc subtracts values
249
+ @original_retry_delay_proc = Backburner.configuration.retry_delay_proc
250
+ Backburner.configure { |config| config.retry_delay_proc = @config_retry_delay_proc }
251
+ end
252
+ after { Backburner.configure { |config| config.retry_delay_proc = @original_retry_delay_proc } }
253
+
254
+ # Rather than compare Procs execute them and compare the output
255
+ it "supports proc retry_delay_proc" do
256
+ assert_equal @override_delay_proc.call(2, 1), resolve_retry_delay_proc(@override_delay_proc).call(2, 1)
257
+ end
258
+
259
+ it "supports classes which respond to queue_retry_delay_proc" do
260
+ job = stub(:queue_retry_delay_proc => @override_delay_proc)
261
+ assert_equal @override_delay_proc.call(2, 1), resolve_retry_delay_proc(job).call(2, 1)
262
+ end
263
+
264
+ it "supports classes which return null queue_retry_delay_proc" do
265
+ job = stub(:queue_retry_delay_proc => nil)
266
+ assert_equal @original_retry_delay_proc.call(2, 1), resolve_retry_delay_proc(job).call(2, 1)
267
+ end
268
+
269
+ it "supports classes which don't respond to queue_retry_delay_proc" do
270
+ job = stub(:fake => true)
271
+ assert_equal @original_retry_delay_proc.call(2, 1), resolve_retry_delay_proc(job).call(2, 1)
272
+ end
273
+
274
+ it "supports default retry_delay_proc for null values" do
275
+ assert_equal @original_retry_delay_proc.call(2, 1), resolve_retry_delay_proc(nil).call(2, 1)
276
+ end
277
+ end # resolve_retry_delay_proc
182
278
  end
data/test/queue_test.rb CHANGED
@@ -44,4 +44,26 @@ describe "Backburner::Queue module" do
44
44
  assert_equal 300, NestedDemo::TestJobB.queue_respond_timeout
45
45
  end
46
46
  end # queue_respond_timeout
47
+
48
+ describe "for queue_max_job_retries assignment method" do
49
+ it "should allow queue max_job_retries to be assigned" do
50
+ NestedDemo::TestJobB.queue_max_job_retries(5)
51
+ assert_equal 5, NestedDemo::TestJobB.queue_max_job_retries
52
+ end
53
+ end # queue_max_job_retries
54
+
55
+ describe "for queue_retry_delay assignment method" do
56
+ it "should allow queue retry_delay to be assigned" do
57
+ NestedDemo::TestJobB.queue_retry_delay(300)
58
+ assert_equal 300, NestedDemo::TestJobB.queue_retry_delay
59
+ end
60
+ end # queue_retry_delay
61
+
62
+ describe "for queue_retry_delay_proc assignment method" do
63
+ it "should allow queue retry_delay_proc to be assigned" do
64
+ retry_delay_proc = lambda { |x, y| x - y }
65
+ NestedDemo::TestJobB.queue_retry_delay_proc(retry_delay_proc)
66
+ assert_equal retry_delay_proc.call(2, 1), NestedDemo::TestJobB.queue_retry_delay_proc.call(2, 1)
67
+ end
68
+ end # queue_retry_delay_proc
47
69
  end # Backburner::Queue
@@ -229,6 +229,33 @@ describe "Backburner::Workers::Simple module" do
229
229
  assert_equal true, $worker_success
230
230
  end
231
231
 
232
+ it "should allow queue override of retries" do
233
+ max_job_retries = TestRetryWithQueueOverridesJob.queue_max_job_retries
234
+ clear_jobs!('foo.bar')
235
+ Backburner.configure do |config|
236
+ # Config should be overridden by queue overrides
237
+ config.max_job_retries = 20
238
+ config.retry_delay = 60
239
+ #config.retry_delay_proc = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) } # default retry_delay_proc
240
+ end
241
+ @worker_class.enqueue TestRetryWithQueueOverridesJob, [max_job_retries], :queue => 'foo.bar'
242
+ out = []
243
+ (max_job_retries + 1).times do
244
+ out << silenced(5) do
245
+ worker = @worker_class.new('foo.bar')
246
+ worker.prepare
247
+ worker.work_one_job
248
+ end
249
+ end
250
+ assert_match(/attempt 1 of 4, retrying in 0/, out.first)
251
+ assert_match(/attempt 2 of 4, retrying in 1/, out[1])
252
+ assert_match(/attempt 3 of 4, retrying in 4/, out[2])
253
+ assert_match(/Completed TestRetryWithQueueOverridesJob/m, out.last)
254
+ refute_match(/failed/, out.last)
255
+ assert_equal 4, $worker_test_count
256
+ assert_equal true, $worker_success
257
+ end
258
+
232
259
  it "should support event hooks without retry" do
233
260
  $hooked_fail_count = 0
234
261
  clear_jobs!('foo.bar.events')
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: backburner
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.5.0
4
+ version: 1.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nathan Esquenazi
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-09-19 00:00:00.000000000 Z
11
+ date: 2021-12-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: beaneater
@@ -171,7 +171,7 @@ homepage: http://github.com/nesquena/backburner
171
171
  licenses:
172
172
  - MIT
173
173
  metadata: {}
174
- post_install_message:
174
+ post_install_message:
175
175
  rdoc_options: []
176
176
  require_paths:
177
177
  - lib
@@ -186,9 +186,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
186
186
  - !ruby/object:Gem::Version
187
187
  version: '0'
188
188
  requirements: []
189
- rubyforge_project:
190
- rubygems_version: 2.6.7
191
- signing_key:
189
+ rubygems_version: 3.0.8
190
+ signing_key:
192
191
  specification_version: 4
193
192
  summary: Reliable beanstalk background job processing made easy for Ruby and Sinatra
194
193
  test_files: