backburner 1.3.1 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.travis.yml +8 -1
- data/CHANGELOG.md +19 -0
- data/README.md +30 -7
- data/backburner.gemspec +1 -2
- data/lib/backburner/configuration.rb +4 -0
- data/lib/backburner/connection.rb +1 -1
- data/lib/backburner/helpers.rb +53 -2
- data/lib/backburner/job.rb +29 -12
- data/lib/backburner/logger.rb +3 -3
- data/lib/backburner/queue.rb +45 -0
- data/lib/backburner/version.rb +1 -1
- data/lib/backburner/worker.rb +8 -5
- data/lib/backburner/workers/threading.rb +48 -14
- data/test/fixtures/test_jobs.rb +39 -0
- data/test/helpers_test.rb +96 -0
- data/test/job_test.rb +54 -16
- data/test/queue_test.rb +22 -0
- data/test/test_helper.rb +3 -3
- data/test/worker_test.rb +29 -0
- data/test/workers/forking_worker_test.rb +2 -1
- data/test/workers/simple_worker_test.rb +29 -2
- data/test/workers/threading_worker_test.rb +42 -6
- data/test/workers/threads_on_fork_worker_test.rb +1 -1
- metadata +12 -21
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 93c9b77199209b894ae963d9474a3234212225021537f99b9fee11433e73f6c4
|
4
|
+
data.tar.gz: 49e775e7a32ccc85a557f27ba89a146617553b45e0a252ddfe12f84d1e501706
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f6e5cdea7a5e3092b8dfc61199ccabf1eb898b3eae4c5645171a7fecc415d7c1c467d1fd173d78732dee6c60bfd2f5928cb728a52e13269bbd951d6d304a2ceb
|
7
|
+
data.tar.gz: e0da0a58b32ea88704e33a4e19a03b3f3b1c8cb63428091743953112841e6b2fc1bf27d5de18d51a8777cad9ee45e069a02a0f9cc132fdf8a0e036bd0379e162
|
data/.travis.yml
CHANGED
@@ -2,6 +2,11 @@
|
|
2
2
|
rvm:
|
3
3
|
- 1.9.3
|
4
4
|
- 2.0.0
|
5
|
+
- 2.1
|
6
|
+
- 2.2
|
7
|
+
- 2.3
|
8
|
+
- 2.4
|
9
|
+
- 2.5
|
5
10
|
- rbx-2
|
6
11
|
before_install:
|
7
12
|
- curl -L https://github.com/kr/beanstalkd/archive/v1.9.tar.gz | tar xz -C /tmp
|
@@ -9,11 +14,13 @@ before_install:
|
|
9
14
|
- make
|
10
15
|
- ./beanstalkd &
|
11
16
|
- cd $TRAVIS_BUILD_DIR
|
17
|
+
- gem update --system
|
18
|
+
- gem update bundler
|
12
19
|
matrix:
|
13
20
|
allow_failures:
|
14
21
|
- rvm: rbx-2
|
22
|
+
- rvm: 2.0.0
|
15
23
|
script:
|
16
|
-
- bundle install
|
17
24
|
- bundle exec rake test
|
18
25
|
gemfile: Gemfile
|
19
26
|
notifications:
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,24 @@
|
|
1
1
|
# CHANGELOG
|
2
2
|
|
3
|
+
## Version 1.6.0 (December 30 2021)
|
4
|
+
|
5
|
+
* TBD (please help backfill)
|
6
|
+
|
7
|
+
## Version 1.5.0 (September 10 2018)
|
8
|
+
|
9
|
+
* TBD
|
10
|
+
|
11
|
+
## Version 1.4.1 (June 10 2017)
|
12
|
+
|
13
|
+
* Fix warning for constant ::Fixnum is deprecated (@amatsuda)
|
14
|
+
|
15
|
+
## Version 1.4.0 (May 13 2017)
|
16
|
+
|
17
|
+
* Fix unit tests to be more consistent (@eltone)
|
18
|
+
* Ensure job supports body hash with symbol keys (@eltone)
|
19
|
+
* Add support for custom serialization formats (@eltone)
|
20
|
+
* Log the params when a job timeout occurs (@nathantsoi)
|
21
|
+
|
3
22
|
## Version 1.3.1 (April 21 2016)
|
4
23
|
|
5
24
|
* Addition of thread-pool-based concurrency (@contentfree)
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
# Backburner
|
1
|
+
# Backburner [](https://travis-ci.org/nesquena/backburner)
|
2
2
|
|
3
|
-
Backburner is a [beanstalkd](
|
3
|
+
Backburner is a [beanstalkd](https://beanstalkd.github.io/)-powered job queue that can handle a very high volume of jobs.
|
4
4
|
You create background jobs and place them on multiple work queues to be processed later.
|
5
5
|
|
6
6
|
Processing background jobs reliably has never been easier than with beanstalkd and Backburner. This gem works with any ruby-based
|
@@ -34,7 +34,7 @@ The real question then is... "Why Beanstalk?".
|
|
34
34
|
|
35
35
|
Illya has an excellent blog post
|
36
36
|
[Scalable Work Queues with Beanstalk](http://www.igvita.com/2010/05/20/scalable-work-queues-with-beanstalk/) and
|
37
|
-
Adam Wiggins posted [an excellent comparison](http://adam.
|
37
|
+
Adam Wiggins posted [an excellent comparison](http://adam.herokuapp.com/past/2010/4/24/beanstalk_a_simple_and_fast_queueing_backend/).
|
38
38
|
|
39
39
|
You will quickly see that **beanstalkd** is an underrated but incredible project that is extremely well-suited as a job queue.
|
40
40
|
Significantly better suited for this task than Redis or a database. Beanstalk is a simple,
|
@@ -65,7 +65,7 @@ In the end, **beanstalk is the ideal job queue** while also being ridiculously e
|
|
65
65
|
|
66
66
|
## Installation
|
67
67
|
|
68
|
-
First, you probably want to [install beanstalkd](
|
68
|
+
First, you probably want to [install beanstalkd](https://beanstalkd.github.io/download.html), which powers the job queues.
|
69
69
|
Depending on your platform, this should be as simple as (for Ubuntu):
|
70
70
|
|
71
71
|
$ sudo apt-get install beanstalkd
|
@@ -88,7 +88,7 @@ Backburner is extremely simple to setup. Just configure basic settings for backb
|
|
88
88
|
|
89
89
|
```ruby
|
90
90
|
Backburner.configure do |config|
|
91
|
-
config.beanstalk_url =
|
91
|
+
config.beanstalk_url = "beanstalk://127.0.0.1"
|
92
92
|
config.tube_namespace = "some.app.production"
|
93
93
|
config.namespace_separator = "."
|
94
94
|
config.on_error = lambda { |e| puts e }
|
@@ -102,6 +102,9 @@ Backburner.configure do |config|
|
|
102
102
|
config.primary_queue = "backburner-jobs"
|
103
103
|
config.priority_labels = { :custom => 50, :useless => 1000 }
|
104
104
|
config.reserve_timeout = nil
|
105
|
+
config.job_serializer_proc = lambda { |body| JSON.dump(body) }
|
106
|
+
config.job_parser_proc = lambda { |body| JSON.parse(body) }
|
107
|
+
|
105
108
|
end
|
106
109
|
```
|
107
110
|
|
@@ -109,7 +112,7 @@ The key options available are:
|
|
109
112
|
|
110
113
|
| Option | Description |
|
111
114
|
| ----------------- | ------------------------------- |
|
112
|
-
| `beanstalk_url` | Address
|
115
|
+
| `beanstalk_url` | Address for beanstalkd connection i.e 'beanstalk://127.0.0.1' |
|
113
116
|
| `tube_namespace` | Prefix used for all tubes related to this backburner queue. |
|
114
117
|
| `namespace_separator` | Separator used for namespace and queue name |
|
115
118
|
| `on_error` | Lambda invoked with the error whenever any job in the system fails. |
|
@@ -123,6 +126,8 @@ The key options available are:
|
|
123
126
|
| `primary_queue` | Primary queue used for a job when an alternate queue is not given. |
|
124
127
|
| `priority_labels` | Hash of named priority definitions for your app. |
|
125
128
|
| `reserve_timeout` | Duration to wait for work from a single server, or nil for forever. |
|
129
|
+
| `job_serializer_proc` | Lambda serializes a job body to a string to write to the task |
|
130
|
+
| `job_parser_proc` | Lambda parses a task body string to a hash |
|
126
131
|
|
127
132
|
## Breaking Changes
|
128
133
|
|
@@ -158,10 +163,25 @@ class NewsletterJob
|
|
158
163
|
1000 # most urgent priority is 0
|
159
164
|
end
|
160
165
|
|
161
|
-
# optional, defaults to respond_timeout
|
166
|
+
# optional, defaults to respond_timeout in config
|
162
167
|
def self.queue_respond_timeout
|
163
168
|
300 # number of seconds before job times out, 0 to avoid timeout. NB: A timeout of 1 second will likely lead to race conditions between Backburner and beanstalkd and should be avoided
|
164
169
|
end
|
170
|
+
|
171
|
+
# optional, defaults to retry_delay_proc in config
|
172
|
+
def self.queue_retry_delay_proc
|
173
|
+
lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 5) }
|
174
|
+
end
|
175
|
+
|
176
|
+
# optional, defaults to retry_delay in config
|
177
|
+
def self.queue_retry_delay
|
178
|
+
5
|
179
|
+
end
|
180
|
+
|
181
|
+
# optional, defaults to max_job_retries in config
|
182
|
+
def self.queue_max_job_retries
|
183
|
+
5
|
184
|
+
end
|
165
185
|
end
|
166
186
|
```
|
167
187
|
|
@@ -508,6 +528,9 @@ end
|
|
508
528
|
|
509
529
|
Now all backburner queue errors will appear on airbrake for deeper inspection.
|
510
530
|
|
531
|
+
If you wish to retry a job without logging an error (for example when handling transient issues in a cloud or service oriented environment),
|
532
|
+
simply raise a `Backburner::Job::RetryJob` error.
|
533
|
+
|
511
534
|
### Logging
|
512
535
|
|
513
536
|
Logging in backburner is rather simple. When a job is run, the log records that. When a job
|
data/backburner.gemspec
CHANGED
@@ -18,10 +18,9 @@ Gem::Specification.new do |s|
|
|
18
18
|
|
19
19
|
s.add_runtime_dependency 'beaneater', '~> 1.0'
|
20
20
|
s.add_runtime_dependency 'dante', '> 0.1.5'
|
21
|
-
s.add_runtime_dependency 'concurrent-ruby', '~> 1.0.1'
|
21
|
+
s.add_runtime_dependency 'concurrent-ruby', '~> 1.0', '>= 1.0.1'
|
22
22
|
|
23
23
|
s.add_development_dependency 'rake'
|
24
24
|
s.add_development_dependency 'minitest', '3.2.0'
|
25
25
|
s.add_development_dependency 'mocha'
|
26
|
-
s.add_development_dependency 'byebug'
|
27
26
|
end
|
@@ -17,6 +17,8 @@ module Backburner
|
|
17
17
|
attr_accessor :primary_queue # the general queue
|
18
18
|
attr_accessor :priority_labels # priority labels
|
19
19
|
attr_accessor :reserve_timeout # duration to wait to reserve on a single server
|
20
|
+
attr_accessor :job_serializer_proc # proc to write the job body to a string
|
21
|
+
attr_accessor :job_parser_proc # proc to parse a job body from a string
|
20
22
|
|
21
23
|
def initialize
|
22
24
|
@beanstalk_url = "beanstalk://127.0.0.1"
|
@@ -34,6 +36,8 @@ module Backburner
|
|
34
36
|
@primary_queue = "backburner-jobs"
|
35
37
|
@priority_labels = PRIORITY_LABELS
|
36
38
|
@reserve_timeout = nil
|
39
|
+
@job_serializer_proc = lambda { |body| body.to_json }
|
40
|
+
@job_parser_proc = lambda { |body| JSON.parse(body) }
|
37
41
|
end
|
38
42
|
|
39
43
|
def namespace_separator=(val)
|
@@ -34,7 +34,7 @@ module Backburner
|
|
34
34
|
def connected?
|
35
35
|
begin
|
36
36
|
!!(@beanstalk && @beanstalk.connection && @beanstalk.connection.connection && !@beanstalk.connection.connection.closed?) # Would be nice if beaneater provided a connected? method
|
37
|
-
rescue
|
37
|
+
rescue
|
38
38
|
false
|
39
39
|
end
|
40
40
|
end
|
data/lib/backburner/helpers.rb
CHANGED
@@ -114,7 +114,7 @@ module Backburner
|
|
114
114
|
resolve_priority(pri.queue_priority)
|
115
115
|
elsif pri.is_a?(String) || pri.is_a?(Symbol) # named priority
|
116
116
|
resolve_priority(Backburner.configuration.priority_labels[pri.to_sym])
|
117
|
-
elsif pri.is_a?(
|
117
|
+
elsif pri.is_a?(Integer) # numerical
|
118
118
|
pri
|
119
119
|
else # default
|
120
120
|
Backburner.configuration.default_priority
|
@@ -131,12 +131,63 @@ module Backburner
|
|
131
131
|
def resolve_respond_timeout(ttr)
|
132
132
|
if ttr.respond_to?(:queue_respond_timeout)
|
133
133
|
resolve_respond_timeout(ttr.queue_respond_timeout)
|
134
|
-
elsif ttr.is_a?(
|
134
|
+
elsif ttr.is_a?(Integer) # numerical
|
135
135
|
ttr
|
136
136
|
else # default
|
137
137
|
Backburner.configuration.respond_timeout
|
138
138
|
end
|
139
139
|
end
|
140
140
|
|
141
|
+
# Resolves max retries based on the value given. Can be integer, a class or nothing
|
142
|
+
#
|
143
|
+
# @example
|
144
|
+
# resolve_max_job_retries(5) => 5
|
145
|
+
# resolve_max_job_retries(FooBar) => <queue max_job_retries>
|
146
|
+
# resolve_max_job_retries(nil) => <default max_job_retries>
|
147
|
+
#
|
148
|
+
def resolve_max_job_retries(retries)
|
149
|
+
if retries.respond_to?(:queue_max_job_retries)
|
150
|
+
resolve_max_job_retries(retries.queue_max_job_retries)
|
151
|
+
elsif retries.is_a?(Integer) # numerical
|
152
|
+
retries
|
153
|
+
else # default
|
154
|
+
Backburner.configuration.max_job_retries
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
# Resolves retry delay based on the value given. Can be integer, a class or nothing
|
159
|
+
#
|
160
|
+
# @example
|
161
|
+
# resolve_retry_delay(5) => 5
|
162
|
+
# resolve_retry_delay(FooBar) => <queue retry_delay>
|
163
|
+
# resolve_retry_delay(nil) => <default retry_delay>
|
164
|
+
#
|
165
|
+
def resolve_retry_delay(delay)
|
166
|
+
if delay.respond_to?(:queue_retry_delay)
|
167
|
+
resolve_retry_delay(delay.queue_retry_delay)
|
168
|
+
elsif delay.is_a?(Integer) # numerical
|
169
|
+
delay
|
170
|
+
else # default
|
171
|
+
Backburner.configuration.retry_delay
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
# Resolves retry delay proc based on the value given. Can be proc, a class or nothing
|
176
|
+
#
|
177
|
+
# @example
|
178
|
+
# resolve_retry_delay_proc(proc) => proc
|
179
|
+
# resolve_retry_delay_proc(FooBar) => <queue retry_delay_proc>
|
180
|
+
# resolve_retry_delay_proc(nil) => <default retry_delay_proc>
|
181
|
+
#
|
182
|
+
def resolve_retry_delay_proc(proc)
|
183
|
+
if proc.respond_to?(:queue_retry_delay_proc)
|
184
|
+
resolve_retry_delay_proc(proc.queue_retry_delay_proc)
|
185
|
+
elsif proc.is_a?(Proc)
|
186
|
+
proc
|
187
|
+
else # default
|
188
|
+
Backburner.configuration.retry_delay_proc
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
141
192
|
end # Helpers
|
142
193
|
end # Backburner
|
data/lib/backburner/job.rb
CHANGED
@@ -7,6 +7,7 @@ module Backburner
|
|
7
7
|
class JobTimeout < RuntimeError; end
|
8
8
|
class JobNotFound < RuntimeError; end
|
9
9
|
class JobFormatInvalid < RuntimeError; end
|
10
|
+
class RetryJob < RuntimeError; end
|
10
11
|
|
11
12
|
attr_accessor :task, :body, :name, :args
|
12
13
|
|
@@ -21,8 +22,9 @@ module Backburner
|
|
21
22
|
def initialize(task)
|
22
23
|
@hooks = Backburner::Hooks
|
23
24
|
@task = task
|
24
|
-
@body = task.body.is_a?(Hash) ? task.body :
|
25
|
-
@name
|
25
|
+
@body = task.body.is_a?(Hash) ? task.body : Backburner.configuration.job_parser_proc.call(task.body)
|
26
|
+
@name = body["class"] || body[:class]
|
27
|
+
@args = body["args"] || body[:args]
|
26
28
|
rescue => ex # Job was not valid format
|
27
29
|
self.bury
|
28
30
|
raise JobFormatInvalid, "Job body could not be parsed: #{ex.inspect}"
|
@@ -42,10 +44,10 @@ module Backburner
|
|
42
44
|
#
|
43
45
|
def process
|
44
46
|
# Invoke before hook and stop if false
|
45
|
-
res = @hooks.invoke_hook_events(
|
47
|
+
res = @hooks.invoke_hook_events(job_name, :before_perform, *args)
|
46
48
|
return false unless res
|
47
49
|
# Execute the job
|
48
|
-
@hooks.around_hook_events(
|
50
|
+
@hooks.around_hook_events(job_name, :around_perform, *args) do
|
49
51
|
# We subtract one to ensure we timeout before beanstalkd does, except if:
|
50
52
|
# a) ttr == 0, to support never timing out
|
51
53
|
# b) ttr == 1, so that we don't accidentally set it to never time out
|
@@ -55,35 +57,50 @@ module Backburner
|
|
55
57
|
end
|
56
58
|
task.delete
|
57
59
|
# Invoke after perform hook
|
58
|
-
@hooks.invoke_hook_events(
|
60
|
+
@hooks.invoke_hook_events(job_name, :after_perform, *args)
|
59
61
|
rescue => e
|
60
|
-
@hooks.invoke_hook_events(
|
62
|
+
@hooks.invoke_hook_events(job_name, :on_failure, e, *args)
|
61
63
|
raise e
|
62
64
|
end
|
63
65
|
|
64
66
|
def bury
|
65
|
-
@hooks.invoke_hook_events(
|
67
|
+
@hooks.invoke_hook_events(job_name, :on_bury, *args)
|
66
68
|
task.bury
|
67
69
|
end
|
68
70
|
|
69
71
|
def retry(count, delay)
|
70
|
-
@hooks.invoke_hook_events(
|
72
|
+
@hooks.invoke_hook_events(job_name, :on_retry, count, delay, *args)
|
71
73
|
task.release(delay: delay)
|
72
74
|
end
|
73
75
|
|
74
|
-
protected
|
75
|
-
|
76
76
|
# Returns the class for the job handler
|
77
77
|
#
|
78
78
|
# @example
|
79
79
|
# job_class # => NewsletterSender
|
80
80
|
#
|
81
81
|
def job_class
|
82
|
-
handler =
|
82
|
+
handler = try_job_class
|
83
83
|
raise(JobNotFound, self.name) unless handler
|
84
84
|
handler
|
85
85
|
end
|
86
86
|
|
87
|
+
protected
|
88
|
+
|
89
|
+
# Attempts to return a constantized job name, otherwise reverts to the name string
|
90
|
+
#
|
91
|
+
# @example
|
92
|
+
# job_name # => "SomeUnknownJob"
|
93
|
+
def job_name
|
94
|
+
handler = try_job_class
|
95
|
+
handler ? handler : self.name
|
96
|
+
end
|
97
|
+
|
98
|
+
def try_job_class
|
99
|
+
constantize(self.name)
|
100
|
+
rescue NameError
|
101
|
+
nil
|
102
|
+
end
|
103
|
+
|
87
104
|
# Timeout job within specified block after given time.
|
88
105
|
#
|
89
106
|
# @example
|
@@ -93,7 +110,7 @@ module Backburner
|
|
93
110
|
begin
|
94
111
|
Timeout::timeout(secs) { yield }
|
95
112
|
rescue Timeout::Error => e
|
96
|
-
raise JobTimeout, "#{name} hit #{secs}s timeout.\nbacktrace: #{e.backtrace}"
|
113
|
+
raise JobTimeout, "#{name}(#{(@args||[]).join(', ')}) hit #{secs}s timeout.\nbacktrace: #{e.backtrace}"
|
97
114
|
end
|
98
115
|
end
|
99
116
|
|
data/lib/backburner/logger.rb
CHANGED
@@ -10,7 +10,7 @@ module Backburner
|
|
10
10
|
# Print out when a job is about to begin
|
11
11
|
def log_job_begin(name, args)
|
12
12
|
log_info "Work job #{name} with #{args.inspect}"
|
13
|
-
|
13
|
+
Thread.current[:job_started_at] = Time.now
|
14
14
|
end
|
15
15
|
|
16
16
|
# Print out when a job completed
|
@@ -24,7 +24,7 @@ module Backburner
|
|
24
24
|
|
25
25
|
# Returns true if the job logging started
|
26
26
|
def job_started_at
|
27
|
-
|
27
|
+
Thread.current[:job_started_at]
|
28
28
|
end
|
29
29
|
|
30
30
|
# Print a message to stdout
|
@@ -50,4 +50,4 @@ module Backburner
|
|
50
50
|
Backburner.configuration.logger
|
51
51
|
end
|
52
52
|
end
|
53
|
-
end
|
53
|
+
end
|
data/lib/backburner/queue.rb
CHANGED
@@ -4,6 +4,9 @@ module Backburner
|
|
4
4
|
base.instance_variable_set(:@queue_name, nil)
|
5
5
|
base.instance_variable_set(:@queue_priority, nil)
|
6
6
|
base.instance_variable_set(:@queue_respond_timeout, nil)
|
7
|
+
base.instance_variable_set(:@queue_max_job_retries, nil)
|
8
|
+
base.instance_variable_set(:@queue_retry_delay, nil)
|
9
|
+
base.instance_variable_set(:@queue_retry_delay_proc, nil)
|
7
10
|
base.instance_variable_set(:@queue_jobs_limit, nil)
|
8
11
|
base.instance_variable_set(:@queue_garbage_limit, nil)
|
9
12
|
base.instance_variable_set(:@queue_retry_limit, nil)
|
@@ -54,6 +57,48 @@ module Backburner
|
|
54
57
|
end
|
55
58
|
end
|
56
59
|
|
60
|
+
# Returns or assigns queue max_job_retries for this job
|
61
|
+
#
|
62
|
+
# @example
|
63
|
+
# queue_max_job_retries 120
|
64
|
+
# @klass.queue_max_job_retries # => 120
|
65
|
+
#
|
66
|
+
def queue_max_job_retries(delay=nil)
|
67
|
+
if delay
|
68
|
+
@queue_max_job_retries = delay
|
69
|
+
else # accessor
|
70
|
+
@queue_max_job_retries
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
# Returns or assigns queue retry_delay for this job
|
75
|
+
#
|
76
|
+
# @example
|
77
|
+
# queue_retry_delay 120
|
78
|
+
# @klass.queue_retry_delay # => 120
|
79
|
+
#
|
80
|
+
def queue_retry_delay(delay=nil)
|
81
|
+
if delay
|
82
|
+
@queue_retry_delay = delay
|
83
|
+
else # accessor
|
84
|
+
@queue_retry_delay
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
# Returns or assigns queue retry_delay_proc for this job
|
89
|
+
#
|
90
|
+
# @example
|
91
|
+
# queue_retry_delay_proc lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 2) }
|
92
|
+
# @klass.queue_retry_delay_proc # => lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 2) }
|
93
|
+
#
|
94
|
+
def queue_retry_delay_proc(proc=nil)
|
95
|
+
if proc
|
96
|
+
@queue_retry_delay_proc = proc
|
97
|
+
else # accessor
|
98
|
+
@queue_retry_delay_proc
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
57
102
|
# Returns or assigns queue parallel active jobs limit (only ThreadsOnFork and Threading workers)
|
58
103
|
#
|
59
104
|
# @example
|
data/lib/backburner/version.rb
CHANGED
data/lib/backburner/worker.rb
CHANGED
@@ -39,7 +39,8 @@ module Backburner
|
|
39
39
|
connection = Backburner::Connection.new(Backburner.configuration.beanstalk_url)
|
40
40
|
connection.retryable do
|
41
41
|
tube = connection.tubes[expand_tube_name(queue || job_class)]
|
42
|
-
|
42
|
+
serialized_data = Backburner.configuration.job_serializer_proc.call(data)
|
43
|
+
response = tube.put(serialized_data, :pri => pri, :delay => delay, :ttr => ttr)
|
43
44
|
end
|
44
45
|
return nil unless Backburner::Hooks.invoke_hook_events(job_class, :after_enqueue, *args)
|
45
46
|
ensure
|
@@ -139,7 +140,7 @@ module Backburner
|
|
139
140
|
rescue Backburner::Job::JobFormatInvalid => e
|
140
141
|
self.log_error self.exception_message(e)
|
141
142
|
rescue => e # Error occurred processing job
|
142
|
-
self.log_error self.exception_message(e)
|
143
|
+
self.log_error self.exception_message(e) unless e.is_a?(Backburner::Job::RetryJob)
|
143
144
|
|
144
145
|
unless job
|
145
146
|
self.log_error "Error occurred before we were able to assign a job. Giving up without retrying!"
|
@@ -149,9 +150,11 @@ module Backburner
|
|
149
150
|
# NB: There's a slight chance here that the connection to beanstalkd has
|
150
151
|
# gone down between the time we reserved / processed the job and here.
|
151
152
|
num_retries = job.stats.releases
|
152
|
-
|
153
|
-
|
154
|
-
|
153
|
+
max_job_retries = resolve_max_job_retries(job.job_class)
|
154
|
+
retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
|
155
|
+
if num_retries < max_job_retries # retry again
|
156
|
+
retry_delay = resolve_retry_delay(job.job_class)
|
157
|
+
delay = resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries) rescue retry_delay
|
155
158
|
job.retry(num_retries + 1, delay)
|
156
159
|
self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
|
157
160
|
else # retries failed, bury
|
@@ -3,9 +3,13 @@ require 'concurrent'
|
|
3
3
|
module Backburner
|
4
4
|
module Workers
|
5
5
|
class Threading < Worker
|
6
|
+
attr_accessor :self_read, :self_write, :exit_on_shutdown
|
7
|
+
|
8
|
+
@shutdown_timeout = 10
|
9
|
+
|
6
10
|
class << self
|
7
|
-
attr_accessor :shutdown
|
8
11
|
attr_accessor :threads_number
|
12
|
+
attr_accessor :shutdown_timeout
|
9
13
|
end
|
10
14
|
|
11
15
|
# Custom initializer just to set @tubes_data
|
@@ -13,6 +17,7 @@ module Backburner
|
|
13
17
|
@tubes_data = {}
|
14
18
|
super
|
15
19
|
self.process_tube_options
|
20
|
+
@exit_on_shutdown = true
|
16
21
|
end
|
17
22
|
|
18
23
|
# Used to prepare job queues before processing jobs.
|
@@ -48,16 +53,19 @@ module Backburner
|
|
48
53
|
connection.on_reconnect = lambda { |conn| conn.tubes.watch!(tube_name) }
|
49
54
|
|
50
55
|
# Make it work jobs using its own connection per thread
|
51
|
-
pool.post(connection)
|
52
|
-
|
56
|
+
pool.post(connection) do |memo_connection|
|
57
|
+
# TODO: use read-write lock?
|
58
|
+
loop do
|
53
59
|
begin
|
54
|
-
|
55
|
-
|
60
|
+
break if @in_shutdown
|
61
|
+
work_one_job(memo_connection)
|
56
62
|
rescue => e
|
57
63
|
log_error("Exception caught in thread pool loop. Continuing. -> #{e.message}\nBacktrace: #{e.backtrace}")
|
58
64
|
end
|
59
|
-
|
60
|
-
|
65
|
+
end
|
66
|
+
|
67
|
+
connection.close
|
68
|
+
end
|
61
69
|
end
|
62
70
|
end
|
63
71
|
|
@@ -111,18 +119,44 @@ module Backburner
|
|
111
119
|
|
112
120
|
# Wait for the shutdown signel
|
113
121
|
def wait_for_shutdown!
|
114
|
-
while
|
115
|
-
|
122
|
+
raise Interrupt while IO.select([self_read])
|
123
|
+
rescue Interrupt
|
124
|
+
shutdown
|
125
|
+
end
|
126
|
+
|
127
|
+
def shutdown_threadpools
|
128
|
+
@thread_pools.each { |_name, pool| pool.shutdown }
|
129
|
+
shutdown_time = Time.now
|
130
|
+
@in_shutdown = true
|
131
|
+
all_shutdown = @thread_pools.all? do |_name, pool|
|
132
|
+
time_to_wait = self.class.shutdown_timeout - (Time.now - shutdown_time).to_i
|
133
|
+
pool.wait_for_termination(time_to_wait) if time_to_wait > 0
|
116
134
|
end
|
135
|
+
rescue Interrupt
|
136
|
+
log_info "graceful shutdown aborted, shutting down immediately"
|
137
|
+
ensure
|
138
|
+
kill unless all_shutdown
|
139
|
+
end
|
117
140
|
|
118
|
-
|
119
|
-
|
120
|
-
@thread_pools.each { |name, pool| pool.kill }
|
141
|
+
def kill
|
142
|
+
@thread_pools.each { |_name, pool| pool.kill unless pool.shutdown? }
|
121
143
|
end
|
122
144
|
|
123
145
|
def shutdown
|
124
|
-
|
125
|
-
|
146
|
+
log_info "beginning graceful worker shutdown"
|
147
|
+
shutdown_threadpools
|
148
|
+
super if @exit_on_shutdown
|
149
|
+
end
|
150
|
+
|
151
|
+
# Registers signal handlers TERM and INT to trigger
|
152
|
+
def register_signal_handlers!
|
153
|
+
@self_read, @self_write = IO.pipe
|
154
|
+
%w[TERM INT].each do |sig|
|
155
|
+
trap(sig) do
|
156
|
+
raise Interrupt if @in_shutdown
|
157
|
+
self_write.puts(sig)
|
158
|
+
end
|
159
|
+
end
|
126
160
|
end
|
127
161
|
end # Threading
|
128
162
|
end # Workers
|
data/test/fixtures/test_jobs.rb
CHANGED
@@ -13,6 +13,24 @@ class TestJob
|
|
13
13
|
def self.perform(x, y); $worker_test_count += x + y; end
|
14
14
|
end
|
15
15
|
|
16
|
+
class TestSlowJob
|
17
|
+
include Backburner::Queue
|
18
|
+
queue_priority :medium
|
19
|
+
queue_respond_timeout 300
|
20
|
+
def self.perform(x, y); sleep 1; $worker_test_count += x + y; end
|
21
|
+
end
|
22
|
+
|
23
|
+
class TestStuckJob
|
24
|
+
include Backburner::Queue
|
25
|
+
queue_priority :medium
|
26
|
+
queue_respond_timeout 300
|
27
|
+
def self.perform(_x, _y)
|
28
|
+
loop do
|
29
|
+
sleep 0.5
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
16
34
|
class TestFailJob
|
17
35
|
include Backburner::Queue
|
18
36
|
def self.perform(x, y); raise RuntimeError; end
|
@@ -36,6 +54,27 @@ class TestConfigurableRetryJob
|
|
36
54
|
end
|
37
55
|
end
|
38
56
|
|
57
|
+
class TestRetryWithQueueOverridesJob
|
58
|
+
include Backburner::Queue
|
59
|
+
def self.perform(retry_count)
|
60
|
+
$worker_test_count += 1
|
61
|
+
raise RuntimeError unless $worker_test_count > retry_count
|
62
|
+
$worker_success = true
|
63
|
+
end
|
64
|
+
|
65
|
+
def self.queue_max_job_retries
|
66
|
+
3
|
67
|
+
end
|
68
|
+
|
69
|
+
def self.queue_retry_delay
|
70
|
+
0
|
71
|
+
end
|
72
|
+
|
73
|
+
def self.queue_retry_delay_proc
|
74
|
+
lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 2) }
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
39
78
|
class TestAsyncJob
|
40
79
|
include Backburner::Performable
|
41
80
|
def self.foo(x, y); $worker_test_count = x * y; end
|
data/test/helpers_test.rb
CHANGED
@@ -179,4 +179,100 @@ describe "Backburner::Helpers module" do
|
|
179
179
|
assert_equal 300, resolve_respond_timeout(nil)
|
180
180
|
end
|
181
181
|
end # resolve_respond_timeout
|
182
|
+
|
183
|
+
describe "for resolve_max_job_retries method" do
|
184
|
+
before do
|
185
|
+
@original_max_job_retries = Backburner.configuration.max_job_retries
|
186
|
+
Backburner.configure { |config| config.max_job_retries = 300 }
|
187
|
+
end
|
188
|
+
after { Backburner.configure { |config| config.max_job_retries = @original_max_job_retries } }
|
189
|
+
|
190
|
+
it "supports fix num max_job_retries" do
|
191
|
+
assert_equal 500, resolve_max_job_retries(500)
|
192
|
+
end
|
193
|
+
|
194
|
+
it "supports classes which respond to queue_max_job_retries" do
|
195
|
+
job = stub(:queue_max_job_retries => 600)
|
196
|
+
assert_equal 600, resolve_max_job_retries(job)
|
197
|
+
end
|
198
|
+
|
199
|
+
it "supports classes which return null queue_max_job_retries" do
|
200
|
+
job = stub(:queue_max_job_retries => nil)
|
201
|
+
assert_equal 300, resolve_max_job_retries(job)
|
202
|
+
end
|
203
|
+
|
204
|
+
it "supports classes which don't respond to queue_max_job_retries" do
|
205
|
+
job = stub(:fake => true)
|
206
|
+
assert_equal 300, resolve_max_job_retries(job)
|
207
|
+
end
|
208
|
+
|
209
|
+
it "supports default max_job_retries for null values" do
|
210
|
+
assert_equal 300, resolve_max_job_retries(nil)
|
211
|
+
end
|
212
|
+
end # resolve_max_job_retries
|
213
|
+
|
214
|
+
describe "for resolve_retry_delay method" do
|
215
|
+
before do
|
216
|
+
@original_retry_delay = Backburner.configuration.retry_delay
|
217
|
+
Backburner.configure { |config| config.retry_delay = 300 }
|
218
|
+
end
|
219
|
+
after { Backburner.configure { |config| config.retry_delay = @original_retry_delay } }
|
220
|
+
|
221
|
+
it "supports fix num retry_delay" do
|
222
|
+
assert_equal 500, resolve_retry_delay(500)
|
223
|
+
end
|
224
|
+
|
225
|
+
it "supports classes which respond to queue_retry_delay" do
|
226
|
+
job = stub(:queue_retry_delay => 600)
|
227
|
+
assert_equal 600, resolve_retry_delay(job)
|
228
|
+
end
|
229
|
+
|
230
|
+
it "supports classes which return null queue_retry_delay" do
|
231
|
+
job = stub(:queue_retry_delay => nil)
|
232
|
+
assert_equal 300, resolve_retry_delay(job)
|
233
|
+
end
|
234
|
+
|
235
|
+
it "supports classes which don't respond to queue_retry_delay" do
|
236
|
+
job = stub(:fake => true)
|
237
|
+
assert_equal 300, resolve_retry_delay(job)
|
238
|
+
end
|
239
|
+
|
240
|
+
it "supports default retry_delay for null values" do
|
241
|
+
assert_equal 300, resolve_retry_delay(nil)
|
242
|
+
end
|
243
|
+
end # resolve_retry_delay
|
244
|
+
|
245
|
+
describe "for resolve_retry_delay_proc method" do
|
246
|
+
before do
|
247
|
+
@config_retry_delay_proc = lambda { |x, y| x + y } # Default config proc adds two values
|
248
|
+
@override_delay_proc = lambda { |x, y| x - y } # Overriden proc subtracts values
|
249
|
+
@original_retry_delay_proc = Backburner.configuration.retry_delay_proc
|
250
|
+
Backburner.configure { |config| config.retry_delay_proc = @config_retry_delay_proc }
|
251
|
+
end
|
252
|
+
after { Backburner.configure { |config| config.retry_delay_proc = @original_retry_delay_proc } }
|
253
|
+
|
254
|
+
# Rather than compare Procs execute them and compare the output
|
255
|
+
it "supports proc retry_delay_proc" do
|
256
|
+
assert_equal @override_delay_proc.call(2, 1), resolve_retry_delay_proc(@override_delay_proc).call(2, 1)
|
257
|
+
end
|
258
|
+
|
259
|
+
it "supports classes which respond to queue_retry_delay_proc" do
|
260
|
+
job = stub(:queue_retry_delay_proc => @override_delay_proc)
|
261
|
+
assert_equal @override_delay_proc.call(2, 1), resolve_retry_delay_proc(job).call(2, 1)
|
262
|
+
end
|
263
|
+
|
264
|
+
it "supports classes which return null queue_retry_delay_proc" do
|
265
|
+
job = stub(:queue_retry_delay_proc => nil)
|
266
|
+
assert_equal @original_retry_delay_proc.call(2, 1), resolve_retry_delay_proc(job).call(2, 1)
|
267
|
+
end
|
268
|
+
|
269
|
+
it "supports classes which don't respond to queue_retry_delay_proc" do
|
270
|
+
job = stub(:fake => true)
|
271
|
+
assert_equal @original_retry_delay_proc.call(2, 1), resolve_retry_delay_proc(job).call(2, 1)
|
272
|
+
end
|
273
|
+
|
274
|
+
it "supports default retry_delay_proc for null values" do
|
275
|
+
assert_equal @original_retry_delay_proc.call(2, 1), resolve_retry_delay_proc(nil).call(2, 1)
|
276
|
+
end
|
277
|
+
end # resolve_retry_delay_proc
|
182
278
|
end
|
data/test/job_test.rb
CHANGED
@@ -15,16 +15,29 @@ describe "Backburner::Job module" do
|
|
15
15
|
describe "for initialize" do
|
16
16
|
describe "with hash" do
|
17
17
|
before do
|
18
|
-
@
|
19
|
-
@task = stub(:body => @task_body, :ttr => 120, :delete => true, :bury => true)
|
18
|
+
@task = stub(:body => task_body, :ttr => 120, :delete => true, :bury => true)
|
20
19
|
end
|
21
20
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
21
|
+
describe "with string keys" do
|
22
|
+
let(:task_body) { { "class" => "NewsletterSender", "args" => ["foo@bar.com", "bar@foo.com"] } }
|
23
|
+
it "should create job with correct task data" do
|
24
|
+
@job = Backburner::Job.new(@task)
|
25
|
+
assert_equal @task, @job.task
|
26
|
+
assert_equal ["class", "args"], @job.body.keys
|
27
|
+
assert_equal task_body["class"], @job.name
|
28
|
+
assert_equal task_body["args"], @job.args
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
describe "with symbol keys" do
|
33
|
+
let(:task_body) { { :class => "NewsletterSender", :args => ["foo@bar.com", "bar@foo.com"] } }
|
34
|
+
it "should create job with correct task data" do
|
35
|
+
@job = Backburner::Job.new(@task)
|
36
|
+
assert_equal @task, @job.task
|
37
|
+
assert_equal [:class, :args], @job.body.keys
|
38
|
+
assert_equal task_body[:class], @job.name
|
39
|
+
assert_equal task_body[:args], @job.args
|
40
|
+
end
|
28
41
|
end
|
29
42
|
end # with hash
|
30
43
|
|
@@ -100,16 +113,41 @@ describe "Backburner::Job module" do
|
|
100
113
|
end # process
|
101
114
|
|
102
115
|
describe "for simple delegation method" do
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
116
|
+
describe "with valid class" do
|
117
|
+
before do
|
118
|
+
@task_body = { "class" => "NestedDemo::TestJobC", "args" => [56] }
|
119
|
+
@task = stub(:body => @task_body, :ttr => 120, :delete => true, :bury => true)
|
120
|
+
@task.expects(:bury).once
|
121
|
+
end
|
122
|
+
|
123
|
+
it "should call bury for task" do
|
124
|
+
@job = Backburner::Job.new(@task)
|
125
|
+
@job.bury
|
126
|
+
end # bury
|
107
127
|
end
|
108
128
|
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
129
|
+
describe "with invalid class" do
|
130
|
+
before do
|
131
|
+
@task_body = { "class" => "AnUnknownClass", "args" => [] }
|
132
|
+
@task = stub(:body => @task_body, :ttr => 120, :delete => true, :bury => true, :release => true)
|
133
|
+
end
|
134
|
+
|
135
|
+
it "should call bury for task" do
|
136
|
+
@task.expects(:bury).once
|
137
|
+
@job = Backburner::Job.new(@task)
|
138
|
+
Backburner::Hooks.expects(:invoke_hook_events)
|
139
|
+
.with("AnUnknownClass", :on_bury, anything)
|
140
|
+
@job.bury
|
141
|
+
end
|
142
|
+
|
143
|
+
it "should call retry for task" do
|
144
|
+
@task.expects(:release).once
|
145
|
+
@job = Backburner::Job.new(@task)
|
146
|
+
Backburner::Hooks.expects(:invoke_hook_events)
|
147
|
+
.with("AnUnknownClass", :on_retry, 0, is_a(Integer), anything)
|
148
|
+
@job.retry(0, 0)
|
149
|
+
end
|
150
|
+
end
|
113
151
|
end # simple delegation
|
114
152
|
|
115
153
|
describe "timing out for various values of ttr" do
|
data/test/queue_test.rb
CHANGED
@@ -44,4 +44,26 @@ describe "Backburner::Queue module" do
|
|
44
44
|
assert_equal 300, NestedDemo::TestJobB.queue_respond_timeout
|
45
45
|
end
|
46
46
|
end # queue_respond_timeout
|
47
|
+
|
48
|
+
describe "for queue_max_job_retries assignment method" do
|
49
|
+
it "should allow queue max_job_retries to be assigned" do
|
50
|
+
NestedDemo::TestJobB.queue_max_job_retries(5)
|
51
|
+
assert_equal 5, NestedDemo::TestJobB.queue_max_job_retries
|
52
|
+
end
|
53
|
+
end # queue_max_job_retries
|
54
|
+
|
55
|
+
describe "for queue_retry_delay assignment method" do
|
56
|
+
it "should allow queue retry_delay to be assigned" do
|
57
|
+
NestedDemo::TestJobB.queue_retry_delay(300)
|
58
|
+
assert_equal 300, NestedDemo::TestJobB.queue_retry_delay
|
59
|
+
end
|
60
|
+
end # queue_retry_delay
|
61
|
+
|
62
|
+
describe "for queue_retry_delay_proc assignment method" do
|
63
|
+
it "should allow queue retry_delay_proc to be assigned" do
|
64
|
+
retry_delay_proc = lambda { |x, y| x - y }
|
65
|
+
NestedDemo::TestJobB.queue_retry_delay_proc(retry_delay_proc)
|
66
|
+
assert_equal retry_delay_proc.call(2, 1), NestedDemo::TestJobB.queue_retry_delay_proc.call(2, 1)
|
67
|
+
end
|
68
|
+
end # queue_retry_delay_proc
|
47
69
|
end # Backburner::Queue
|
data/test/test_helper.rb
CHANGED
@@ -6,7 +6,7 @@ begin
|
|
6
6
|
rescue LoadError
|
7
7
|
require 'mocha'
|
8
8
|
end
|
9
|
-
|
9
|
+
$LOAD_PATH.unshift File.expand_path("lib")
|
10
10
|
require 'backburner'
|
11
11
|
require File.expand_path('../helpers/templogger', __FILE__)
|
12
12
|
|
@@ -92,12 +92,12 @@ class MiniTest::Spec
|
|
92
92
|
end
|
93
93
|
|
94
94
|
# pop_one_job(tube_name)
|
95
|
-
def pop_one_job(tube_name=Backburner.configuration.primary_queue
|
95
|
+
def pop_one_job(tube_name=Backburner.configuration.primary_queue)
|
96
96
|
tube_name = [Backburner.configuration.tube_namespace, tube_name].join(".")
|
97
97
|
connection = beanstalk_connection
|
98
98
|
connection.tubes.watch!(tube_name)
|
99
99
|
silenced(3) { @res = connection.tubes.reserve }
|
100
|
-
yield @res,
|
100
|
+
yield @res, Backburner.configuration.job_parser_proc.call(@res.body)
|
101
101
|
ensure
|
102
102
|
connection.close if connection
|
103
103
|
end
|
data/test/worker_test.rb
CHANGED
@@ -125,4 +125,33 @@ describe "Backburner::Worker module" do
|
|
125
125
|
assert_equal ['baz', 'bam'], worker.tube_names
|
126
126
|
end
|
127
127
|
end # tube_names
|
128
|
+
|
129
|
+
describe "for custom serialization" do
|
130
|
+
before do
|
131
|
+
Backburner.configure do |config|
|
132
|
+
@old_parser = config.job_parser_proc
|
133
|
+
@old_serializer = config.job_serializer_proc
|
134
|
+
config.job_parser_proc = lambda { |body| Marshal.load(body) }
|
135
|
+
config.job_serializer_proc = lambda { |body| Marshal.dump(body) }
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
after do
|
140
|
+
clear_jobs!('test-plain')
|
141
|
+
Backburner.configure do |config|
|
142
|
+
config.job_parser_proc = @old_parser
|
143
|
+
config.job_serializer_proc = @old_serializer
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
it "should support enqueuing a job" do
|
148
|
+
Backburner::Worker.enqueue TestPlainJob, [7, 9], :ttr => 100, :pri => 2000
|
149
|
+
pop_one_job("test-plain") do |job, body|
|
150
|
+
assert_equal "TestPlainJob", body[:class]
|
151
|
+
assert_equal [7, 9], body[:args]
|
152
|
+
assert_equal 100, job.ttr
|
153
|
+
assert_equal 2000, job.pri
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end # custom serialization
|
128
157
|
end # Backburner::Worker
|
@@ -11,7 +11,7 @@ describe "Backburner::Workers::Forking module" do
|
|
11
11
|
describe "for prepare method" do
|
12
12
|
it "should make tube names array always unique to avoid duplication" do
|
13
13
|
worker = @worker_class.new(["foo", "demo.test.foo"])
|
14
|
-
worker.prepare
|
14
|
+
capture_stdout { worker.prepare }
|
15
15
|
assert_equal ["demo.test.foo"], worker.tube_names
|
16
16
|
end
|
17
17
|
|
@@ -96,6 +96,7 @@ describe "Backburner::Workers::Forking module" do
|
|
96
96
|
|
97
97
|
after do
|
98
98
|
@templogger.close
|
99
|
+
Backburner.configuration.logger = nil
|
99
100
|
clear_jobs!('response')
|
100
101
|
clear_jobs!('bar.foo.1', 'bar.foo.2', 'bar.foo.3', 'bar.foo.4', 'bar.foo.5')
|
101
102
|
end
|
@@ -11,7 +11,7 @@ describe "Backburner::Workers::Simple module" do
|
|
11
11
|
describe "for prepare method" do
|
12
12
|
it "should make tube names array always unique to avoid duplication" do
|
13
13
|
worker = @worker_class.new(["foo", "demo.test.foo"])
|
14
|
-
worker.prepare
|
14
|
+
capture_stdout { worker.prepare }
|
15
15
|
assert_equal ["demo.test.foo"], worker.tube_names
|
16
16
|
end
|
17
17
|
|
@@ -229,6 +229,33 @@ describe "Backburner::Workers::Simple module" do
|
|
229
229
|
assert_equal true, $worker_success
|
230
230
|
end
|
231
231
|
|
232
|
+
it "should allow queue override of retries" do
|
233
|
+
max_job_retries = TestRetryWithQueueOverridesJob.queue_max_job_retries
|
234
|
+
clear_jobs!('foo.bar')
|
235
|
+
Backburner.configure do |config|
|
236
|
+
# Config should be overridden by queue overrides
|
237
|
+
config.max_job_retries = 20
|
238
|
+
config.retry_delay = 60
|
239
|
+
#config.retry_delay_proc = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) } # default retry_delay_proc
|
240
|
+
end
|
241
|
+
@worker_class.enqueue TestRetryWithQueueOverridesJob, [max_job_retries], :queue => 'foo.bar'
|
242
|
+
out = []
|
243
|
+
(max_job_retries + 1).times do
|
244
|
+
out << silenced(5) do
|
245
|
+
worker = @worker_class.new('foo.bar')
|
246
|
+
worker.prepare
|
247
|
+
worker.work_one_job
|
248
|
+
end
|
249
|
+
end
|
250
|
+
assert_match(/attempt 1 of 4, retrying in 0/, out.first)
|
251
|
+
assert_match(/attempt 2 of 4, retrying in 1/, out[1])
|
252
|
+
assert_match(/attempt 3 of 4, retrying in 4/, out[2])
|
253
|
+
assert_match(/Completed TestRetryWithQueueOverridesJob/m, out.last)
|
254
|
+
refute_match(/failed/, out.last)
|
255
|
+
assert_equal 4, $worker_test_count
|
256
|
+
assert_equal true, $worker_success
|
257
|
+
end
|
258
|
+
|
232
259
|
it "should support event hooks without retry" do
|
233
260
|
$hooked_fail_count = 0
|
234
261
|
clear_jobs!('foo.bar.events')
|
@@ -309,7 +336,7 @@ describe "Backburner::Workers::Simple module" do
|
|
309
336
|
worker = @worker_class.new('foo.bar')
|
310
337
|
connection = mock('connection')
|
311
338
|
worker.expects(:reserve_job).with(connection).returns(stub_everything('job'))
|
312
|
-
worker.work_one_job(connection)
|
339
|
+
capture_stdout { worker.work_one_job(connection) }
|
313
340
|
end
|
314
341
|
|
315
342
|
after do
|
@@ -6,24 +6,25 @@ describe "Backburner::Workers::Threading worker" do
|
|
6
6
|
before do
|
7
7
|
Backburner.default_queues.clear
|
8
8
|
@worker_class = Backburner::Workers::Threading
|
9
|
+
@worker_class.shutdown_timeout = 2
|
9
10
|
end
|
10
11
|
|
11
12
|
describe "for prepare method" do
|
12
13
|
it "should make tube names array always unique to avoid duplication" do
|
13
14
|
worker = @worker_class.new(["foo", "demo.test.foo"])
|
14
|
-
worker.prepare
|
15
|
+
capture_stdout { worker.prepare }
|
15
16
|
assert_equal ["demo.test.foo"], worker.tube_names
|
16
17
|
end
|
17
18
|
|
18
19
|
it 'creates a thread pool per queue' do
|
19
20
|
worker = @worker_class.new(%w(foo bar))
|
20
|
-
|
21
|
+
capture_stdout { worker.prepare }
|
21
22
|
assert_equal 2, worker.instance_variable_get("@thread_pools").keys.size
|
22
23
|
end
|
23
24
|
|
24
25
|
it 'uses Concurrent.processor_count if no custom thread count is provided' do
|
25
26
|
worker = @worker_class.new("foo")
|
26
|
-
|
27
|
+
capture_stdout { worker.prepare }
|
27
28
|
assert_equal ::Concurrent.processor_count, worker.instance_variable_get("@thread_pools")["demo.test.foo"].max_length
|
28
29
|
end
|
29
30
|
end # prepare
|
@@ -54,15 +55,50 @@ describe "Backburner::Workers::Threading worker" do
|
|
54
55
|
before do
|
55
56
|
@worker = @worker_class.new(["foo:3"])
|
56
57
|
capture_stdout { @worker.prepare }
|
58
|
+
$worker_test_count = 0
|
59
|
+
$worker_success = false
|
57
60
|
end
|
58
61
|
|
59
62
|
it 'runs work_on_job per thread' do
|
60
63
|
clear_jobs!("foo")
|
61
64
|
job_count=10
|
62
|
-
|
63
|
-
@
|
64
|
-
|
65
|
+
# TestJob adds the given arguments together and then to $worker_test_count
|
66
|
+
job_count.times { @worker_class.enqueue TestJob, [1, 0], :queue => "foo" }
|
67
|
+
capture_stdout do
|
68
|
+
@worker.start(false) # don't wait for shutdown
|
69
|
+
sleep 0.5 # Wait for threads to do their work
|
70
|
+
end
|
65
71
|
assert_equal job_count, $worker_test_count
|
66
72
|
end
|
67
73
|
end # working a queue
|
74
|
+
|
75
|
+
describe 'shutting down' do
|
76
|
+
before do
|
77
|
+
@thread_count = 3
|
78
|
+
@worker = @worker_class.new(["threaded-shutdown:#{@thread_count}"])
|
79
|
+
@worker.exit_on_shutdown = false
|
80
|
+
$worker_test_count = 0
|
81
|
+
clear_jobs!("threaded-shutdown")
|
82
|
+
end
|
83
|
+
|
84
|
+
it 'gracefully exits and completes all in-flight jobs' do
|
85
|
+
6.times { @worker_class.enqueue TestSlowJob, [1, 0], :queue => "threaded-shutdown" }
|
86
|
+
Thread.new { sleep 0.1; @worker.self_write.puts("TERM") }
|
87
|
+
capture_stdout do
|
88
|
+
@worker.start
|
89
|
+
end
|
90
|
+
|
91
|
+
assert_equal @thread_count, $worker_test_count
|
92
|
+
end
|
93
|
+
|
94
|
+
it 'forces an exit when a job is stuck' do
|
95
|
+
6.times { @worker_class.enqueue TestStuckJob, [1, 0], :queue => "threaded-shutdown" }
|
96
|
+
Thread.new { sleep 0.1; @worker.self_write.puts("TERM") }
|
97
|
+
capture_stdout do
|
98
|
+
@worker.start
|
99
|
+
end
|
100
|
+
|
101
|
+
assert_equal 0, $worker_test_count
|
102
|
+
end
|
103
|
+
end
|
68
104
|
end
|
@@ -70,7 +70,7 @@ describe "Backburner::Workers::ThreadsOnFork module" do
|
|
70
70
|
|
71
71
|
it "should make tube names array always unique to avoid duplication" do
|
72
72
|
worker = @worker_class.new(["foo", "demo.test.foo"])
|
73
|
-
worker.prepare
|
73
|
+
capture_stdout { worker.prepare }
|
74
74
|
assert_equal ["demo.test.foo"], worker.tube_names
|
75
75
|
end
|
76
76
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: backburner
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nathan Esquenazi
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-12-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: beaneater
|
@@ -43,6 +43,9 @@ dependencies:
|
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.0'
|
48
|
+
- - ">="
|
46
49
|
- !ruby/object:Gem::Version
|
47
50
|
version: 1.0.1
|
48
51
|
type: :runtime
|
@@ -50,6 +53,9 @@ dependencies:
|
|
50
53
|
version_requirements: !ruby/object:Gem::Requirement
|
51
54
|
requirements:
|
52
55
|
- - "~>"
|
56
|
+
- !ruby/object:Gem::Version
|
57
|
+
version: '1.0'
|
58
|
+
- - ">="
|
53
59
|
- !ruby/object:Gem::Version
|
54
60
|
version: 1.0.1
|
55
61
|
- !ruby/object:Gem::Dependency
|
@@ -94,20 +100,6 @@ dependencies:
|
|
94
100
|
- - ">="
|
95
101
|
- !ruby/object:Gem::Version
|
96
102
|
version: '0'
|
97
|
-
- !ruby/object:Gem::Dependency
|
98
|
-
name: byebug
|
99
|
-
requirement: !ruby/object:Gem::Requirement
|
100
|
-
requirements:
|
101
|
-
- - ">="
|
102
|
-
- !ruby/object:Gem::Version
|
103
|
-
version: '0'
|
104
|
-
type: :development
|
105
|
-
prerelease: false
|
106
|
-
version_requirements: !ruby/object:Gem::Requirement
|
107
|
-
requirements:
|
108
|
-
- - ">="
|
109
|
-
- !ruby/object:Gem::Version
|
110
|
-
version: '0'
|
111
103
|
description: Beanstalk background job processing made easy
|
112
104
|
email:
|
113
105
|
- nesquena@gmail.com
|
@@ -179,7 +171,7 @@ homepage: http://github.com/nesquena/backburner
|
|
179
171
|
licenses:
|
180
172
|
- MIT
|
181
173
|
metadata: {}
|
182
|
-
post_install_message:
|
174
|
+
post_install_message:
|
183
175
|
rdoc_options: []
|
184
176
|
require_paths:
|
185
177
|
- lib
|
@@ -194,9 +186,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
194
186
|
- !ruby/object:Gem::Version
|
195
187
|
version: '0'
|
196
188
|
requirements: []
|
197
|
-
|
198
|
-
|
199
|
-
signing_key:
|
189
|
+
rubygems_version: 3.0.8
|
190
|
+
signing_key:
|
200
191
|
specification_version: 4
|
201
192
|
summary: Reliable beanstalk background job processing made easy for Ruby and Sinatra
|
202
193
|
test_files:
|