backburner 1.7.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +53 -4
- data/HOOKS.md +23 -14
- data/README.md +102 -23
- data/bin/backburner +1 -1
- data/lib/active_job/queue_adapters/backburner_adapter.rb +19 -5
- data/lib/backburner/cli.rb +1 -1
- data/lib/backburner/configuration.rb +7 -0
- data/lib/backburner/connection.rb +4 -2
- data/lib/backburner/enqueue_pool.rb +90 -0
- data/lib/backburner/helpers.rb +77 -22
- data/lib/backburner/job.rb +46 -7
- data/lib/backburner/logger.rb +9 -0
- data/lib/backburner/performable.rb +29 -4
- data/lib/backburner/queue.rb +37 -19
- data/lib/backburner/version.rb +1 -1
- data/lib/backburner/worker.rb +150 -25
- data/lib/backburner/workers/forking.rb +4 -2
- data/lib/backburner/workers/simple.rb +5 -0
- data/lib/backburner/workers/threading.rb +17 -7
- data/lib/backburner/workers/threads_on_fork.rb +59 -23
- data/lib/backburner.rb +7 -1
- metadata +40 -66
- data/.gitignore +0 -17
- data/.travis.yml +0 -29
- data/CONTRIBUTING.md +0 -37
- data/Gemfile +0 -4
- data/Rakefile +0 -17
- data/TODO +0 -4
- data/backburner.gemspec +0 -28
- data/circle.yml +0 -3
- data/examples/custom.rb +0 -25
- data/examples/demo.rb +0 -60
- data/examples/god.rb +0 -46
- data/examples/hooked.rb +0 -87
- data/examples/retried.rb +0 -31
- data/examples/simple.rb +0 -43
- data/examples/stress.rb +0 -31
- data/test/async_proxy_test.rb +0 -36
- data/test/back_burner_test.rb +0 -88
- data/test/connection_test.rb +0 -179
- data/test/fixtures/active_jobs.rb +0 -11
- data/test/fixtures/hooked.rb +0 -122
- data/test/fixtures/test_fork_jobs.rb +0 -72
- data/test/fixtures/test_forking_jobs.rb +0 -56
- data/test/fixtures/test_jobs.rb +0 -87
- data/test/fixtures/test_queue_settings.rb +0 -14
- data/test/helpers/templogger.rb +0 -22
- data/test/helpers_test.rb +0 -278
- data/test/hooks_test.rb +0 -112
- data/test/job_test.rb +0 -185
- data/test/logger_test.rb +0 -44
- data/test/performable_test.rb +0 -88
- data/test/queue_test.rb +0 -69
- data/test/test_helper.rb +0 -125
- data/test/worker_test.rb +0 -157
- data/test/workers/forking_worker_test.rb +0 -181
- data/test/workers/simple_worker_test.rb +0 -350
- data/test/workers/threading_worker_test.rb +0 -104
- data/test/workers/threads_on_fork_worker_test.rb +0 -484
data/lib/backburner/worker.rb
CHANGED
|
@@ -9,6 +9,10 @@ module Backburner
|
|
|
9
9
|
include Backburner::Helpers
|
|
10
10
|
include Backburner::Logger
|
|
11
11
|
|
|
12
|
+
# The longest a worker that doesn't have a job's class waits to hand the job out again (its TTR
|
|
13
|
+
# otherwise), so a worker that has it picks it up soon after a rollout
|
|
14
|
+
MAX_CLASS_NOT_FOUND_DELAY = 60
|
|
15
|
+
|
|
12
16
|
# Backburner::Worker.known_queue_classes
|
|
13
17
|
# List of known_queue_classes
|
|
14
18
|
class << self
|
|
@@ -17,7 +21,8 @@ module Backburner
|
|
|
17
21
|
end
|
|
18
22
|
|
|
19
23
|
# Enqueues a job to be processed later by a worker.
|
|
20
|
-
# Options: `pri` (priority), `delay` (delay in secs), `ttr` (time to respond),
|
|
24
|
+
# Options: `pri` (priority), `delay` (delay in secs, rounded up to whole seconds), `ttr` (time to respond),
|
|
25
|
+
# `queue` (queue name)
|
|
21
26
|
#
|
|
22
27
|
# @raise [Beaneater::NotConnected] If beanstalk fails to connect.
|
|
23
28
|
# @example
|
|
@@ -25,7 +30,7 @@ module Backburner
|
|
|
25
30
|
#
|
|
26
31
|
def self.enqueue(job_class, args=[], opts={})
|
|
27
32
|
pri = resolve_priority(opts[:pri] || job_class)
|
|
28
|
-
delay =
|
|
33
|
+
delay = whole_seconds(opts[:delay])
|
|
29
34
|
ttr = resolve_respond_timeout(opts[:ttr] || job_class)
|
|
30
35
|
res = Backburner::Hooks.invoke_hook_events(job_class, :before_enqueue, *args)
|
|
31
36
|
|
|
@@ -34,18 +39,15 @@ module Backburner
|
|
|
34
39
|
data = { :class => job_class.name, :args => args }
|
|
35
40
|
queue = opts[:queue] && (Proc === opts[:queue] ? opts[:queue].call(job_class) : opts[:queue])
|
|
36
41
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
connection = Backburner::Connection.new(Backburner.configuration.beanstalk_url)
|
|
42
|
+
response = nil
|
|
43
|
+
Backburner::EnqueuePool.current.with_connection do |connection|
|
|
40
44
|
connection.retryable do
|
|
41
45
|
tube = connection.tubes[expand_tube_name(queue || job_class)]
|
|
42
46
|
serialized_data = Backburner.configuration.job_serializer_proc.call(data)
|
|
43
47
|
response = tube.put(serialized_data, :pri => pri, :delay => delay, :ttr => ttr)
|
|
44
48
|
end
|
|
45
|
-
return nil unless Backburner::Hooks.invoke_hook_events(job_class, :after_enqueue, *args)
|
|
46
|
-
ensure
|
|
47
|
-
connection.close if connection
|
|
48
49
|
end
|
|
50
|
+
return nil unless Backburner::Hooks.invoke_hook_events(job_class, :after_enqueue, *args)
|
|
49
51
|
|
|
50
52
|
response
|
|
51
53
|
end
|
|
@@ -71,6 +73,7 @@ module Backburner
|
|
|
71
73
|
# @example
|
|
72
74
|
# Worker.new(['test.job'])
|
|
73
75
|
def initialize(tube_names=nil)
|
|
76
|
+
@existing_tube_names = nil # set by all_existing_queues, read by expand_tube_name
|
|
74
77
|
@connection = new_connection
|
|
75
78
|
@tube_names = self.process_tube_names(tube_names)
|
|
76
79
|
register_signal_handlers!
|
|
@@ -102,9 +105,12 @@ module Backburner
|
|
|
102
105
|
|
|
103
106
|
# Triggers this worker to shutdown
|
|
104
107
|
def shutdown
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
+
# Logs from another thread, since a trap handler can't take a logger's lock, and waits for it (a
|
|
109
|
+
# second at most): the process exits right after, and an unjoined thread never got to run
|
|
110
|
+
# ceiling: if the signal lands while the main thread holds the logger's lock, the line is lost after a
|
|
111
|
+
# second; handling the signal outside the trap (the self-pipe Threading uses) would lift it
|
|
112
|
+
Thread.new { log_info 'Worker exiting...' }.join(1)
|
|
113
|
+
ensure
|
|
108
114
|
Kernel.exit
|
|
109
115
|
end
|
|
110
116
|
|
|
@@ -134,7 +140,7 @@ module Backburner
|
|
|
134
140
|
end
|
|
135
141
|
|
|
136
142
|
self.log_job_begin(job.name, job.args)
|
|
137
|
-
job.process
|
|
143
|
+
wrap_in_rails_reloader { job.process }
|
|
138
144
|
self.log_job_end(job.name)
|
|
139
145
|
|
|
140
146
|
rescue Backburner::Job::JobFormatInvalid => e
|
|
@@ -149,17 +155,45 @@ module Backburner
|
|
|
149
155
|
|
|
150
156
|
# NB: There's a slight chance here that the connection to beanstalkd has
|
|
151
157
|
# gone down between the time we reserved / processed the job and here.
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
158
|
+
class_missing = false
|
|
159
|
+
begin
|
|
160
|
+
job_class = begin
|
|
161
|
+
job.job_class
|
|
162
|
+
rescue Backburner::Job::JobNotFound # missing, or its file failed to load: see below
|
|
163
|
+
nil
|
|
164
|
+
end
|
|
165
|
+
if job_class.nil? && job.job_class_missing?
|
|
166
|
+
# This worker doesn't have its class (renamed or removed, or a worker deployed before the code
|
|
167
|
+
# that enqueues it): give it back for a worker that has it rather than burying it, once per TTR
|
|
168
|
+
# (at most a minute) as beanstalkd would, without on_error or the failure hooks (as before) or
|
|
169
|
+
# using up its retries. A class whose file fails to load goes the normal failure path instead.
|
|
170
|
+
# ceiling: beanstalkd still counts the release, so once a worker has the class, a failure there
|
|
171
|
+
# gets fewer retries; counting these releases apart would need a marker in the job body.
|
|
172
|
+
class_missing = true
|
|
173
|
+
delay = [job.ttr || queue_config.respond_timeout, MAX_CLASS_NOT_FOUND_DELAY].min
|
|
174
|
+
job.task.release(:delay => delay)
|
|
175
|
+
self.log_warn "#{job.name}: class not found; released for #{delay}s for a worker that has it"
|
|
176
|
+
return
|
|
177
|
+
end
|
|
178
|
+
num_retries = job.stats.releases
|
|
179
|
+
max_job_retries = resolve_max_job_retries(job_class)
|
|
180
|
+
retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
|
|
181
|
+
if num_retries < max_job_retries # retry again
|
|
182
|
+
retry_delay = resolve_retry_delay(job_class)
|
|
183
|
+
delay = resolve_retry_delay_proc(job_class).call(retry_delay, num_retries) rescue retry_delay
|
|
184
|
+
delay = whole_seconds(delay)
|
|
185
|
+
job.retry(num_retries + 1, delay)
|
|
186
|
+
self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
|
|
187
|
+
else # retries failed, bury
|
|
188
|
+
job.bury
|
|
189
|
+
self.log_job_end(job.name, "#{retry_status}, burying") if job_started_at
|
|
190
|
+
end
|
|
191
|
+
rescue Beaneater::NotFoundError, Beaneater::NotConnected
|
|
192
|
+
# Its TTR ran out first, or the connection was lost: beanstalkd releases a reserved job when
|
|
193
|
+
# its TTR runs out or its client disconnects (unless the server restarted and lost it).
|
|
194
|
+
# (An error raised in a rescue clause isn't caught by the method's other rescues, hence this one.)
|
|
195
|
+
self.log_error "#{job.name} was no longer reserved (its TTR ran out, or the connection was lost), so it was neither retried nor buried"
|
|
196
|
+
return if class_missing # on_error isn't called for a class this worker lacks
|
|
163
197
|
end
|
|
164
198
|
|
|
165
199
|
handle_error(e, job.name, job.args, job)
|
|
@@ -173,6 +207,36 @@ module Backburner
|
|
|
173
207
|
Connection.new(Backburner.configuration.beanstalk_url) { |conn| Backburner::Hooks.invoke_hook_events(self, :on_reconnect, conn) }
|
|
174
208
|
end
|
|
175
209
|
|
|
210
|
+
# Runs the block (a job's processing) in the Rails app's reloader when there is one, as Rails does for Active Job
|
|
211
|
+
# jobs (whose own wrap, inside this one, then does nothing): each job starts with fresh CurrentAttributes, gives
|
|
212
|
+
# back its database connections when it's done and, in development, runs the app's latest code. On Rails 7.0+ the
|
|
213
|
+
# wrap reports the errors that leave it to Rails.error (with the source "job.backburner" on 7.1+); a RetryJob, a
|
|
214
|
+
# request to retry rather than a failure, and a SystemExit (a TERM or INT the worker stops on, or a job calling
|
|
215
|
+
# exit), which Rails 7.2+ would report too, are raised again outside it instead.
|
|
216
|
+
# ceiling: config.on_error and the on_retry and on_bury hooks run after the wrap ends, so they don't see the job's
|
|
217
|
+
# CurrentAttributes, and a database connection they use is outside the executor; running work_one_job's failure
|
|
218
|
+
# handling inside the wrap would lift it. In development, a Forking or ThreadsOnFork parent never runs the wrap, so
|
|
219
|
+
# after a file change each child reloads the app once.
|
|
220
|
+
def wrap_in_rails_reloader
|
|
221
|
+
app = ::Rails.application if defined?(::Rails) && ::Rails.respond_to?(:application)
|
|
222
|
+
return yield unless app.respond_to?(:reloader)
|
|
223
|
+
deferred = nil
|
|
224
|
+
work = proc do
|
|
225
|
+
begin
|
|
226
|
+
yield
|
|
227
|
+
rescue Backburner::Job::RetryJob, SystemExit => e
|
|
228
|
+
deferred = e
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
# Active Support 7.1+ take the source the executor reports errors with; older ones take no options
|
|
232
|
+
if ActiveSupport.gem_version >= Gem::Version.new("7.1")
|
|
233
|
+
app.reloader.wrap(source: "job.backburner", &work)
|
|
234
|
+
else
|
|
235
|
+
app.reloader.wrap(&work)
|
|
236
|
+
end
|
|
237
|
+
raise deferred if deferred
|
|
238
|
+
end
|
|
239
|
+
|
|
176
240
|
# Reserve a job from the watched queues
|
|
177
241
|
def reserve_job(conn, reserve_timeout = Backburner.configuration.reserve_timeout)
|
|
178
242
|
Backburner::Job.new(conn.tubes.reserve(reserve_timeout))
|
|
@@ -181,11 +245,49 @@ module Backburner
|
|
|
181
245
|
# Returns a list of all tubes known within the system
|
|
182
246
|
# Filtered for tubes that match the known prefix
|
|
183
247
|
def all_existing_queues
|
|
184
|
-
known_queues = Backburner::Worker.known_queue_classes.map(&:queue)
|
|
185
|
-
|
|
248
|
+
known_queues = Backburner::Worker.known_queue_classes.select(&:name).map(&:queue) # an anonymous class can't be a job
|
|
249
|
+
prefix = expanded_tube_prefix # the start of every tube name enqueueing with this configuration gives
|
|
250
|
+
namespace = queue_config.tube_namespace
|
|
251
|
+
cut = namespace && "#{namespace}#{queue_config.namespace_separator}".include?(':') # every name is the prefix
|
|
252
|
+
existing_tubes = self.connection.tubes.all.map(&:name).select do |tube| # only the names this configuration can give
|
|
253
|
+
cut ? tube == prefix : tube.start_with?(prefix) && tube[prefix.size..-1] !~ /[A-Z_]/ # the rest is dasherized
|
|
254
|
+
end
|
|
255
|
+
# ceiling: a server tube named exactly like a known queue name is still expanded like that name, as before;
|
|
256
|
+
# the workers' prepare keeping found tubes apart from queue names would lift it
|
|
257
|
+
@existing_tube_names = existing_tubes - known_queues - [queue_config.primary_queue]
|
|
186
258
|
existing_tubes + known_queues + [queue_config.primary_queue]
|
|
187
259
|
end
|
|
188
260
|
|
|
261
|
+
# Leaves the server's tube names all_existing_queues found as they are: expanding one again can change it
|
|
262
|
+
# ("MyApp.default" would become "MyApp.my-app.default")
|
|
263
|
+
def expand_tube_name(tube)
|
|
264
|
+
@existing_tube_names && @existing_tube_names.include?(tube) ? tube : super
|
|
265
|
+
end
|
|
266
|
+
public :expand_tube_name
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
# Merges the options the block gives for each known queue class (its queue_jobs_limit and so on)
|
|
270
|
+
# into its tube's entry in tubes_data, for tubes that are there. A class that declares its tube
|
|
271
|
+
# overrides the tube's options; a subclass sharing its parent's tube only fills in the ones still
|
|
272
|
+
# unset. Anonymous classes are skipped.
|
|
273
|
+
def merge_tube_settings(tubes_data)
|
|
274
|
+
owners, sharers = Backburner::Worker.known_queue_classes.select(&:name).partition { |queue| !shares_parent_tube?(queue) }
|
|
275
|
+
[[owners, true], [sharers, false]].each do |classes, override|
|
|
276
|
+
classes.each do |queue|
|
|
277
|
+
data = tubes_data[expand_tube_name(queue)]
|
|
278
|
+
next if data.nil?
|
|
279
|
+
data.merge!(yield(queue)) { |_key, current, setting| (override ? setting.nil? : !current.nil?) ? current : setting }
|
|
280
|
+
end
|
|
281
|
+
end
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
# Whether a queue class uses the same tube as its closest named Backburner::Queue ancestor
|
|
285
|
+
def shares_parent_tube?(queue)
|
|
286
|
+
return false unless queue.respond_to?(:superclass)
|
|
287
|
+
parent = queue.superclass
|
|
288
|
+
parent = parent.superclass while parent.is_a?(Backburner::Queue::ClassMethods) && parent.name.nil?
|
|
289
|
+
parent.is_a?(Backburner::Queue::ClassMethods) && expand_tube_name(parent) == expand_tube_name(queue)
|
|
290
|
+
end
|
|
189
291
|
|
|
190
292
|
# Handles an error according to custom definition
|
|
191
293
|
# Used when processing a job that errors out
|
|
@@ -199,6 +301,8 @@ module Backburner
|
|
|
199
301
|
error_handler.call(e, name, args, job)
|
|
200
302
|
end
|
|
201
303
|
end
|
|
304
|
+
rescue => handler_error # an error reporter that fails (can't reach its service, say) mustn't stop the worker
|
|
305
|
+
log_error "The on_error handler failed for #{name}: #{exception_message(handler_error)}"
|
|
202
306
|
end
|
|
203
307
|
|
|
204
308
|
# Normalizes tube names given array of tube_names
|
|
@@ -217,5 +321,26 @@ module Backburner
|
|
|
217
321
|
trap('TERM') { shutdown }
|
|
218
322
|
trap('INT') { shutdown }
|
|
219
323
|
end
|
|
324
|
+
|
|
325
|
+
# Whether a forked child disables the GC as it exits (see with_gc_disabled_on_exit): only on CRuby 3.3
|
|
326
|
+
def self.gc_disabled_on_exit?(engine = RUBY_ENGINE, version = RUBY_VERSION)
|
|
327
|
+
engine == "ruby" && version.start_with?("3.3.")
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
# Runs the body of a forked child, then, on Ruby 3.3, disables the GC before the child exits by any
|
|
331
|
+
# path that tears the interpreter down: returning, an exception, or Kernel.exit (as the TERM and INT
|
|
332
|
+
# traps do). Kernel.exit! skips the ensure, and the teardown with it. The child's at_exit handlers
|
|
333
|
+
# (its own and those it inherited) then run with the GC off.
|
|
334
|
+
# ceiling: works around a Ruby 3.3 bug: a child forked while another thread of its parent runs
|
|
335
|
+
# finalizers can crash as it exits ("[BUG] lazy sweeping underway when freeing object space").
|
|
336
|
+
# GC.disable finishes the lazy sweep first. Fixed in Ruby 3.4 by ruby/ruby 868d63f0a3 (not
|
|
337
|
+
# backported to 3.3, and not seen on 3.1 or 3.2); this can go once Ruby 3.3 support is dropped.
|
|
338
|
+
# Limits: an at_exit handler that calls GC.enable itself can still crash, and a TERM that lands while
|
|
339
|
+
# a ThreadsOnFork child is starting its job threads can leave one of them running during at_exit.
|
|
340
|
+
def with_gc_disabled_on_exit
|
|
341
|
+
yield
|
|
342
|
+
ensure
|
|
343
|
+
GC.disable if Worker.gc_disabled_on_exit?
|
|
344
|
+
end
|
|
220
345
|
end # Worker
|
|
221
346
|
end # Backburner
|
|
@@ -24,6 +24,11 @@ module Backburner
|
|
|
24
24
|
prepare
|
|
25
25
|
loop { work_one_job }
|
|
26
26
|
end
|
|
27
|
+
|
|
28
|
+
# A reconnected connection only watches the default tube: watch ours again
|
|
29
|
+
def on_reconnect(conn)
|
|
30
|
+
conn.tubes.watch!(*self.tube_names)
|
|
31
|
+
end
|
|
27
32
|
end # Basic
|
|
28
33
|
end # Workers
|
|
29
34
|
end # Backburner
|
|
@@ -7,6 +7,9 @@ module Backburner
|
|
|
7
7
|
|
|
8
8
|
@shutdown_timeout = 10
|
|
9
9
|
|
|
10
|
+
# The longest a pool thread waits in reserve at a time (beanstalkd takes whole seconds)
|
|
11
|
+
SHUTDOWN_CHECK_INTERVAL = 1
|
|
12
|
+
|
|
10
13
|
class << self
|
|
11
14
|
attr_accessor :threads_number
|
|
12
15
|
attr_accessor :shutdown_timeout
|
|
@@ -15,6 +18,7 @@ module Backburner
|
|
|
15
18
|
# Custom initializer just to set @tubes_data
|
|
16
19
|
def initialize(*args)
|
|
17
20
|
@tubes_data = {}
|
|
21
|
+
@in_shutdown = false
|
|
18
22
|
super
|
|
19
23
|
self.process_tube_options
|
|
20
24
|
@exit_on_shutdown = true
|
|
@@ -108,13 +112,7 @@ module Backburner
|
|
|
108
112
|
# If the tube is known by existing beanstalkd queue, but not by class - skip it
|
|
109
113
|
#
|
|
110
114
|
def process_tube_options
|
|
111
|
-
|
|
112
|
-
next if @tubes_data[expand_tube_name(queue)].nil?
|
|
113
|
-
queue_settings = {
|
|
114
|
-
:threads => queue.queue_jobs_limit
|
|
115
|
-
}
|
|
116
|
-
@tubes_data[expand_tube_name(queue)].merge!(queue_settings){|k, v1, v2| v2.nil? ? v1 : v2 }
|
|
117
|
-
end
|
|
115
|
+
merge_tube_settings(@tubes_data) { |queue| { :threads => queue.queue_jobs_limit } }
|
|
118
116
|
end
|
|
119
117
|
|
|
120
118
|
# Wait for the shutdown signel
|
|
@@ -158,6 +156,18 @@ module Backburner
|
|
|
158
156
|
end
|
|
159
157
|
end
|
|
160
158
|
end
|
|
159
|
+
|
|
160
|
+
protected
|
|
161
|
+
|
|
162
|
+
# Reserves for at most SHUTDOWN_CHECK_INTERVAL seconds, so an idle thread sees a shutdown
|
|
163
|
+
# and stops instead of waiting in reserve until shutdown_timeout kills it
|
|
164
|
+
# ceiling: each idle thread sends beanstalkd a reserve once a second; waking the waiting
|
|
165
|
+
# reserves at shutdown instead would avoid that polling
|
|
166
|
+
def reserve_job(conn, reserve_timeout = Backburner.configuration.reserve_timeout)
|
|
167
|
+
timeout = reserve_timeout.to_s.to_i if reserve_timeout # as beanstalkd reads what beaneater sends: "5", 2.5, 5.seconds
|
|
168
|
+
timeout = SHUTDOWN_CHECK_INTERVAL if timeout.nil? || timeout < 0 || timeout > SHUTDOWN_CHECK_INTERVAL
|
|
169
|
+
super(conn, timeout)
|
|
170
|
+
end
|
|
161
171
|
end # Threading
|
|
162
172
|
end # Workers
|
|
163
173
|
end # Backburner
|
|
@@ -69,6 +69,7 @@ module Backburner
|
|
|
69
69
|
# Custom initializer just to set @tubes_data
|
|
70
70
|
def initialize(*args)
|
|
71
71
|
@tubes_data = {}
|
|
72
|
+
@watching_tube = nil
|
|
72
73
|
super
|
|
73
74
|
self.process_tube_options
|
|
74
75
|
end
|
|
@@ -108,14 +109,8 @@ module Backburner
|
|
|
108
109
|
# If the tube is known by existing beanstalkd queue, but not by class - skip it
|
|
109
110
|
#
|
|
110
111
|
def process_tube_options
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
queue_settings = {
|
|
114
|
-
:threads => queue.queue_jobs_limit,
|
|
115
|
-
:garbage => queue.queue_garbage_limit,
|
|
116
|
-
:retries => queue.queue_retry_limit
|
|
117
|
-
}
|
|
118
|
-
@tubes_data[expand_tube_name(queue)].merge!(queue_settings){|k, v1, v2| v2.nil? ? v1 : v2 }
|
|
112
|
+
merge_tube_settings(@tubes_data) do |queue|
|
|
113
|
+
{ :threads => queue.queue_jobs_limit, :garbage => queue.queue_garbage_limit, :retries => queue.queue_retry_limit }
|
|
119
114
|
end
|
|
120
115
|
end
|
|
121
116
|
|
|
@@ -140,18 +135,37 @@ module Backburner
|
|
|
140
135
|
end
|
|
141
136
|
end
|
|
142
137
|
|
|
138
|
+
# After a child fails, the parent waits before forking the next one: MIN_RESTART_DELAY seconds,
|
|
139
|
+
# doubling while children keep failing (beanstalkd is down, say) up to MAX_RESTART_DELAY, and
|
|
140
|
+
# starting over once a child is recycled (99) or runs for MAX_RESTART_DELAY seconds
|
|
141
|
+
MIN_RESTART_DELAY = 1
|
|
142
|
+
MAX_RESTART_DELAY = 30
|
|
143
|
+
|
|
143
144
|
# Make the fork and create a thread to watch the child process
|
|
144
145
|
# The exit code '99' means that the fork exited because of the garbage limit
|
|
145
146
|
# Any other code is an error
|
|
146
147
|
def fork_and_watch(name)
|
|
147
148
|
create_thread(name) do |tube_name|
|
|
149
|
+
restart_delay = 0
|
|
148
150
|
until self.class.shutdown
|
|
151
|
+
forked_at = monotonic_time
|
|
149
152
|
pid = fork_tube(tube_name)
|
|
150
153
|
_, status = wait_for_process(pid)
|
|
151
154
|
|
|
152
155
|
# 99 = garbaged
|
|
153
|
-
if status.exitstatus
|
|
154
|
-
|
|
156
|
+
if status.exitstatus == 99
|
|
157
|
+
restart_delay = 0
|
|
158
|
+
elsif self.class.shutdown # stopped by SIGTERM or SIGINT, or killed by finish_forks: not forked again
|
|
159
|
+
log_info(status.exitstatus == 0 ? "Tube #{tube_name} exited normally." : "Tube #{tube_name} stopped.")
|
|
160
|
+
else
|
|
161
|
+
ran_for = monotonic_time - forked_at
|
|
162
|
+
restart_delay = ran_for >= MAX_RESTART_DELAY ? MIN_RESTART_DELAY : (restart_delay * 2).clamp(MIN_RESTART_DELAY, MAX_RESTART_DELAY)
|
|
163
|
+
if status.exitstatus == 0 # stopped by a signal sent to the child alone, or a job called exit
|
|
164
|
+
log_warn("Tube #{tube_name} exited with code 0. Forking it again in #{restart_delay}s.")
|
|
165
|
+
else
|
|
166
|
+
log_error("Catastrophic failure: tube #{tube_name} exited with code #{status.exitstatus}. Forking it again in #{restart_delay}s.")
|
|
167
|
+
end
|
|
168
|
+
Kernel.sleep(restart_delay)
|
|
155
169
|
end
|
|
156
170
|
end
|
|
157
171
|
end
|
|
@@ -182,22 +196,37 @@ module Backburner
|
|
|
182
196
|
@runs = 0
|
|
183
197
|
|
|
184
198
|
if @threads_number == 1
|
|
199
|
+
# Each forked child needs its own socket: the parent's connection is shared by every fork
|
|
200
|
+
self.connection = new_connection
|
|
185
201
|
watch_tube(name)
|
|
186
202
|
run_while_can
|
|
187
203
|
else
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
204
|
+
failed = false
|
|
205
|
+
threads = []
|
|
206
|
+
begin
|
|
207
|
+
@threads_number.times do
|
|
208
|
+
threads << create_thread do
|
|
209
|
+
begin
|
|
210
|
+
conn = new_connection
|
|
211
|
+
watch_tube(name, conn)
|
|
212
|
+
run_while_can(conn)
|
|
213
|
+
rescue Exception
|
|
214
|
+
failed = true
|
|
215
|
+
raise
|
|
216
|
+
ensure
|
|
217
|
+
conn.close if conn
|
|
218
|
+
end
|
|
197
219
|
end
|
|
198
220
|
end
|
|
221
|
+
# Wait for these threads only: others can outlive them (the timeout gem's, since Ruby 3.2)
|
|
222
|
+
sleep 0.1 while threads.any?(&:alive?)
|
|
223
|
+
rescue Exception # TERM, INT or an error: on Ruby 3.3 the child runs its at_exit handlers with the GC
|
|
224
|
+
# off (see with_gc_disabled_on_exit), so its threads mustn't keep working jobs meanwhile
|
|
225
|
+
threads.each(&:kill) if Worker.gc_disabled_on_exit?
|
|
226
|
+
raise
|
|
199
227
|
end
|
|
200
|
-
|
|
228
|
+
# Fail like a single-threaded child whose connection raised, so the parent backs off
|
|
229
|
+
Kernel.exit!(1) if failed
|
|
201
230
|
end
|
|
202
231
|
|
|
203
232
|
coolest_exit
|
|
@@ -233,6 +262,11 @@ module Backburner
|
|
|
233
262
|
Thread.new(*args, &block)
|
|
234
263
|
end
|
|
235
264
|
|
|
265
|
+
# Seconds on a clock that only moves forward. Easy to test
|
|
266
|
+
def monotonic_time
|
|
267
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
268
|
+
end
|
|
269
|
+
|
|
236
270
|
# Wait for a specific process. Easy to test
|
|
237
271
|
def wait_for_process(pid)
|
|
238
272
|
out = Process.wait2(pid)
|
|
@@ -245,9 +279,11 @@ module Backburner
|
|
|
245
279
|
# never shutdown
|
|
246
280
|
def fork_it(&blk)
|
|
247
281
|
pid = Kernel.fork do
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
282
|
+
with_gc_disabled_on_exit do
|
|
283
|
+
self.class.is_child = true
|
|
284
|
+
$0 = "[ThreadsOnFork worker] parent: #{Process.ppid}"
|
|
285
|
+
blk.call
|
|
286
|
+
end
|
|
251
287
|
end
|
|
252
288
|
self.class.child_pids << pid
|
|
253
289
|
pid
|
data/lib/backburner.rb
CHANGED
|
@@ -7,6 +7,7 @@ require 'backburner/helpers'
|
|
|
7
7
|
require 'backburner/configuration'
|
|
8
8
|
require 'backburner/logger'
|
|
9
9
|
require 'backburner/connection'
|
|
10
|
+
require 'backburner/enqueue_pool'
|
|
10
11
|
require 'backburner/hooks'
|
|
11
12
|
require 'backburner/performable'
|
|
12
13
|
require 'backburner/worker'
|
|
@@ -15,7 +16,12 @@ require 'backburner/workers/forking'
|
|
|
15
16
|
require 'backburner/workers/threads_on_fork'
|
|
16
17
|
require 'backburner/workers/threading'
|
|
17
18
|
require 'backburner/queue'
|
|
18
|
-
|
|
19
|
+
# On Rails 7.2+ (which added AbstractAdapter) use this gem's Active Job adapter; older Rails autoload theirs.
|
|
20
|
+
# Not when Rails' autoload has already loaded the adapter or is loading it (an adapter file requires backburner).
|
|
21
|
+
if defined?(ActiveJob::QueueAdapters::AbstractAdapter) && (ActiveJob::QueueAdapters.autoload?(:BackburnerAdapter) ||
|
|
22
|
+
!ActiveJob::QueueAdapters.constants(false).include?(:BackburnerAdapter))
|
|
23
|
+
require 'active_job/queue_adapters/backburner_adapter'
|
|
24
|
+
end
|
|
19
25
|
|
|
20
26
|
module Backburner
|
|
21
27
|
class << self
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: backburner
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.9.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Nathan Esquenazi
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-01
|
|
11
|
+
date: 2026-11-01 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: beaneater
|
|
@@ -58,6 +58,20 @@ dependencies:
|
|
|
58
58
|
- - ">="
|
|
59
59
|
- !ruby/object:Gem::Version
|
|
60
60
|
version: 1.0.1
|
|
61
|
+
- !ruby/object:Gem::Dependency
|
|
62
|
+
name: logger
|
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
|
64
|
+
requirements:
|
|
65
|
+
- - ">="
|
|
66
|
+
- !ruby/object:Gem::Version
|
|
67
|
+
version: '0'
|
|
68
|
+
type: :runtime
|
|
69
|
+
prerelease: false
|
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
71
|
+
requirements:
|
|
72
|
+
- - ">="
|
|
73
|
+
- !ruby/object:Gem::Version
|
|
74
|
+
version: '0'
|
|
61
75
|
- !ruby/object:Gem::Dependency
|
|
62
76
|
name: rake
|
|
63
77
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -86,6 +100,20 @@ dependencies:
|
|
|
86
100
|
- - '='
|
|
87
101
|
- !ruby/object:Gem::Version
|
|
88
102
|
version: 5.1.0
|
|
103
|
+
- !ruby/object:Gem::Dependency
|
|
104
|
+
name: mutex_m
|
|
105
|
+
requirement: !ruby/object:Gem::Requirement
|
|
106
|
+
requirements:
|
|
107
|
+
- - ">="
|
|
108
|
+
- !ruby/object:Gem::Version
|
|
109
|
+
version: '0'
|
|
110
|
+
type: :development
|
|
111
|
+
prerelease: false
|
|
112
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
113
|
+
requirements:
|
|
114
|
+
- - ">="
|
|
115
|
+
- !ruby/object:Gem::Version
|
|
116
|
+
version: '0'
|
|
89
117
|
- !ruby/object:Gem::Dependency
|
|
90
118
|
name: mocha
|
|
91
119
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -136,32 +164,18 @@ executables:
|
|
|
136
164
|
extensions: []
|
|
137
165
|
extra_rdoc_files: []
|
|
138
166
|
files:
|
|
139
|
-
- ".gitignore"
|
|
140
|
-
- ".travis.yml"
|
|
141
167
|
- CHANGELOG.md
|
|
142
|
-
- CONTRIBUTING.md
|
|
143
|
-
- Gemfile
|
|
144
168
|
- HOOKS.md
|
|
145
169
|
- LICENSE
|
|
146
170
|
- README.md
|
|
147
|
-
- Rakefile
|
|
148
|
-
- TODO
|
|
149
|
-
- backburner.gemspec
|
|
150
171
|
- bin/backburner
|
|
151
|
-
- circle.yml
|
|
152
|
-
- examples/custom.rb
|
|
153
|
-
- examples/demo.rb
|
|
154
|
-
- examples/god.rb
|
|
155
|
-
- examples/hooked.rb
|
|
156
|
-
- examples/retried.rb
|
|
157
|
-
- examples/simple.rb
|
|
158
|
-
- examples/stress.rb
|
|
159
172
|
- lib/active_job/queue_adapters/backburner_adapter.rb
|
|
160
173
|
- lib/backburner.rb
|
|
161
174
|
- lib/backburner/async_proxy.rb
|
|
162
175
|
- lib/backburner/cli.rb
|
|
163
176
|
- lib/backburner/configuration.rb
|
|
164
177
|
- lib/backburner/connection.rb
|
|
178
|
+
- lib/backburner/enqueue_pool.rb
|
|
165
179
|
- lib/backburner/helpers.rb
|
|
166
180
|
- lib/backburner/hooks.rb
|
|
167
181
|
- lib/backburner/job.rb
|
|
@@ -175,32 +189,14 @@ files:
|
|
|
175
189
|
- lib/backburner/workers/simple.rb
|
|
176
190
|
- lib/backburner/workers/threading.rb
|
|
177
191
|
- lib/backburner/workers/threads_on_fork.rb
|
|
178
|
-
|
|
179
|
-
- test/back_burner_test.rb
|
|
180
|
-
- test/connection_test.rb
|
|
181
|
-
- test/fixtures/active_jobs.rb
|
|
182
|
-
- test/fixtures/hooked.rb
|
|
183
|
-
- test/fixtures/test_fork_jobs.rb
|
|
184
|
-
- test/fixtures/test_forking_jobs.rb
|
|
185
|
-
- test/fixtures/test_jobs.rb
|
|
186
|
-
- test/fixtures/test_queue_settings.rb
|
|
187
|
-
- test/helpers/templogger.rb
|
|
188
|
-
- test/helpers_test.rb
|
|
189
|
-
- test/hooks_test.rb
|
|
190
|
-
- test/job_test.rb
|
|
191
|
-
- test/logger_test.rb
|
|
192
|
-
- test/performable_test.rb
|
|
193
|
-
- test/queue_test.rb
|
|
194
|
-
- test/test_helper.rb
|
|
195
|
-
- test/worker_test.rb
|
|
196
|
-
- test/workers/forking_worker_test.rb
|
|
197
|
-
- test/workers/simple_worker_test.rb
|
|
198
|
-
- test/workers/threading_worker_test.rb
|
|
199
|
-
- test/workers/threads_on_fork_worker_test.rb
|
|
200
|
-
homepage: http://github.com/nesquena/backburner
|
|
192
|
+
homepage: https://github.com/nesquena/backburner
|
|
201
193
|
licenses:
|
|
202
194
|
- MIT
|
|
203
|
-
metadata:
|
|
195
|
+
metadata:
|
|
196
|
+
source_code_uri: https://github.com/nesquena/backburner
|
|
197
|
+
changelog_uri: https://github.com/nesquena/backburner/blob/master/CHANGELOG.md
|
|
198
|
+
bug_tracker_uri: https://github.com/nesquena/backburner/issues
|
|
199
|
+
rubygems_mfa_required: 'true'
|
|
204
200
|
post_install_message:
|
|
205
201
|
rdoc_options: []
|
|
206
202
|
require_paths:
|
|
@@ -209,37 +205,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
209
205
|
requirements:
|
|
210
206
|
- - ">="
|
|
211
207
|
- !ruby/object:Gem::Version
|
|
212
|
-
version: '
|
|
208
|
+
version: '2.7'
|
|
213
209
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
214
210
|
requirements:
|
|
215
211
|
- - ">="
|
|
216
212
|
- !ruby/object:Gem::Version
|
|
217
213
|
version: '0'
|
|
218
214
|
requirements: []
|
|
219
|
-
rubygems_version: 3.
|
|
215
|
+
rubygems_version: 3.5.22
|
|
220
216
|
signing_key:
|
|
221
217
|
specification_version: 4
|
|
222
218
|
summary: Reliable beanstalk background job processing made easy for Ruby and Sinatra
|
|
223
|
-
test_files:
|
|
224
|
-
- test/async_proxy_test.rb
|
|
225
|
-
- test/back_burner_test.rb
|
|
226
|
-
- test/connection_test.rb
|
|
227
|
-
- test/fixtures/active_jobs.rb
|
|
228
|
-
- test/fixtures/hooked.rb
|
|
229
|
-
- test/fixtures/test_fork_jobs.rb
|
|
230
|
-
- test/fixtures/test_forking_jobs.rb
|
|
231
|
-
- test/fixtures/test_jobs.rb
|
|
232
|
-
- test/fixtures/test_queue_settings.rb
|
|
233
|
-
- test/helpers/templogger.rb
|
|
234
|
-
- test/helpers_test.rb
|
|
235
|
-
- test/hooks_test.rb
|
|
236
|
-
- test/job_test.rb
|
|
237
|
-
- test/logger_test.rb
|
|
238
|
-
- test/performable_test.rb
|
|
239
|
-
- test/queue_test.rb
|
|
240
|
-
- test/test_helper.rb
|
|
241
|
-
- test/worker_test.rb
|
|
242
|
-
- test/workers/forking_worker_test.rb
|
|
243
|
-
- test/workers/simple_worker_test.rb
|
|
244
|
-
- test/workers/threading_worker_test.rb
|
|
245
|
-
- test/workers/threads_on_fork_worker_test.rb
|
|
219
|
+
test_files: []
|