backburner 1.6.1 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +45 -0
- data/.github/workflows/github-release.yml +58 -0
- data/.github/workflows/release.yml +221 -0
- data/CHANGELOG.md +33 -0
- data/Gemfile +4 -1
- data/HOOKS.md +23 -14
- data/README.md +32 -9
- data/backburner.gemspec +6 -1
- data/bin/backburner +1 -1
- data/lib/active_job/queue_adapters/backburner_adapter.rb +41 -0
- data/lib/backburner/configuration.rb +7 -0
- data/lib/backburner/connection.rb +4 -2
- data/lib/backburner/enqueue_pool.rb +88 -0
- data/lib/backburner/helpers.rb +77 -22
- data/lib/backburner/job.rb +46 -7
- data/lib/backburner/logger.rb +9 -0
- data/lib/backburner/performable.rb +1 -1
- data/lib/backburner/queue.rb +37 -19
- data/lib/backburner/version.rb +1 -1
- data/lib/backburner/worker.rb +112 -21
- data/lib/backburner/workers/forking.rb +4 -2
- data/lib/backburner/workers/simple.rb +5 -0
- data/lib/backburner/workers/threading.rb +1 -7
- data/lib/backburner/workers/threads_on_fork.rb +58 -23
- data/lib/backburner.rb +7 -0
- data/test/active_job_adapter_test.rb +24 -0
- data/test/back_burner_test.rb +13 -0
- data/test/connection_test.rb +22 -0
- data/test/fixtures/active_job_app.rb +53 -0
- data/test/fixtures/active_jobs.rb +11 -0
- data/test/fixtures/autoload/test_broken_name_job.rb +4 -0
- data/test/fixtures/autoload/test_broken_no_method_job.rb +4 -0
- data/test/fixtures/autoload/test_broken_require_job.rb +4 -0
- data/test/fixtures/autoload/test_broken_runtime_job.rb +4 -0
- data/test/fixtures/autoload/test_broken_syntax_job.rb +4 -0
- data/test/fixtures/autoload/test_broken_timeout_job.rb +4 -0
- data/test/fixtures/autoload/test_same_name_job.rb +5 -0
- data/test/fixtures/test_fork_jobs.rb +5 -0
- data/test/fixtures/test_jobs.rb +53 -0
- data/test/fixtures/test_queue_settings.rb +41 -1
- data/test/helpers/templogger.rb +1 -1
- data/test/helpers_test.rb +184 -0
- data/test/job_test.rb +22 -0
- data/test/logger_test.rb +27 -0
- data/test/performable_test.rb +50 -0
- data/test/queue_test.rb +63 -0
- data/test/test_helper.rb +59 -5
- data/test/worker_test.rb +346 -1
- data/test/workers/forking_worker_test.rb +19 -1
- data/test/workers/simple_worker_test.rb +272 -0
- data/test/workers/threading_worker_test.rb +22 -1
- data/test/workers/threads_on_fork_worker_test.rb +279 -12
- metadata +87 -8
- data/.travis.yml +0 -29
- data/circle.yml +0 -3
|
@@ -61,6 +61,18 @@ describe "Backburner::Workers::ThreadsOnFork module" do
|
|
|
61
61
|
worker = @worker_class.new
|
|
62
62
|
assert_equal(worker.instance_variable_get("@tubes_data")['demo.test.job-settings-override'], { :threads => 10, :garbage => 1000, :retries => 2 })
|
|
63
63
|
end
|
|
64
|
+
|
|
65
|
+
it "should take a shared tube's settings from the class that owns the tube, not from subclasses" do
|
|
66
|
+
tubes_data = @worker_class.new.instance_variable_get("@tubes_data")
|
|
67
|
+
assert_equal({ :threads => 5, :garbage => 100, :retries => 3 }, tubes_data['demo.test.tube-limits'])
|
|
68
|
+
assert_equal({ :threads => 3, :garbage => nil, :retries => nil }, tubes_data['demo.test.tube-limits-testtubelimitslambdajob'])
|
|
69
|
+
assert_equal({ :threads => 4, :garbage => nil, :retries => nil }, tubes_data['demo.test.tube-limits-testtubelimitslambdachildjob'])
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
it "should fill in the settings a shared tube's class leaves unset from its subclasses" do
|
|
73
|
+
tubes_data = @worker_class.new.instance_variable_get("@tubes_data")
|
|
74
|
+
assert_equal({ :threads => 3, :garbage => 50, :retries => 2 }, tubes_data['demo.test.tube-few-limits'])
|
|
75
|
+
end
|
|
64
76
|
end
|
|
65
77
|
|
|
66
78
|
describe "for prepare method" do
|
|
@@ -145,23 +157,147 @@ describe "Backburner::Workers::ThreadsOnFork module" do
|
|
|
145
157
|
refute_match(/Catastrophic failure/, out)
|
|
146
158
|
end
|
|
147
159
|
|
|
148
|
-
it "fork_and_watch thread should log
|
|
149
|
-
process_exit = stub('process_exit')
|
|
150
|
-
process_exit.expects(:exitstatus).twice.returns(0)
|
|
160
|
+
it "fork_and_watch thread should log a normal exit if exitstatus is 0 on shutdown" do
|
|
161
|
+
process_exit = stub('process_exit', :exitstatus => 0)
|
|
151
162
|
worker = @worker_class.new(%(foo))
|
|
152
163
|
worker.expects(:wait_for_process).with(12).returns([nil, process_exit])
|
|
153
164
|
|
|
154
165
|
wc = @worker_class
|
|
155
|
-
# TODO: Is there a best way do do this?
|
|
156
166
|
worker.define_singleton_method :fork_it do
|
|
157
167
|
wc.shutdown = true
|
|
158
168
|
12
|
|
159
169
|
end
|
|
160
170
|
def worker.create_thread(*args, &block); block.call(*args) end
|
|
161
171
|
out = silenced(2) { worker.start(false) }
|
|
162
|
-
assert_match(/
|
|
172
|
+
assert_match(/Tube demo\.test\.foo exited normally\./, out)
|
|
173
|
+
refute_match(/Catastrophic failure/, out)
|
|
163
174
|
end
|
|
164
175
|
|
|
176
|
+
describe "restarting a failed child" do
|
|
177
|
+
before do
|
|
178
|
+
@worker = @worker_class.new(%(foo))
|
|
179
|
+
def @worker.create_thread(*args, &block); block.call(*args) end
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# Runs fork_and_watch over children exiting with these codes, expecting it to sleep these delays in
|
|
183
|
+
# turn, and returns what it logged
|
|
184
|
+
def expect_restart_delays(codes, delays)
|
|
185
|
+
wc = @worker_class
|
|
186
|
+
left = codes.size
|
|
187
|
+
@worker.define_singleton_method(:fork_it) { left -= 1; wc.shutdown = true if left.zero?; 12 }
|
|
188
|
+
@worker.stubs(:wait_for_process).with(12).returns(*codes.map { |code| [12, stub(:exitstatus => code)] })
|
|
189
|
+
restarts = sequence('restarts')
|
|
190
|
+
delays.each { |delay| Kernel.expects(:sleep).with(delay).in_sequence(restarts) }
|
|
191
|
+
silenced(2) { @worker.start(false) }
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
it "should wait longer before each fork while children keep failing, up to MAX_RESTART_DELAY" do
|
|
195
|
+
expect_restart_delays([1] * 7, [1, 2, 4, 8, 16, 30])
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
it "should fork again at once after a recycled child and start the delays over" do
|
|
199
|
+
expect_restart_delays([1, 1, 99, 1, 1], [1, 2, 1])
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
it "should log an error if exitstatus is != 99, and fork again" do
|
|
203
|
+
out = expect_restart_delays([1, 99], [1])
|
|
204
|
+
assert_match(/Catastrophic failure: tube demo\.test\.foo exited with code 1\. Forking it again in 1s\./, out)
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
it "should log a child that stops while shutting down without forking it again" do
|
|
208
|
+
[nil, 1].each do |code| # killed by finish_forks, or exited with an error
|
|
209
|
+
@worker_class.shutdown = false
|
|
210
|
+
out = expect_restart_delays([code], [])
|
|
211
|
+
assert_match(/Tube demo\.test\.foo stopped\./, out)
|
|
212
|
+
refute_match(/Catastrophic failure|Forking it again/, out)
|
|
213
|
+
end
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
it "should log the warning with a logger that has no warn method" do
|
|
217
|
+
logged = []
|
|
218
|
+
logger = Object.new
|
|
219
|
+
logger.define_singleton_method(:info) { |msg| logged << [:info, msg] }
|
|
220
|
+
logger.define_singleton_method(:error) { |msg| logged << [:error, msg] }
|
|
221
|
+
Backburner.configure { |config| config.logger = logger }
|
|
222
|
+
expect_restart_delays([0, 1], [1])
|
|
223
|
+
assert_includes logged, [:error, "Tube demo.test.foo exited with code 0. Forking it again in 1s."]
|
|
224
|
+
ensure
|
|
225
|
+
Backburner.configure { |config| config.logger = nil }
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
it "should warn and wait before forking again after a child exits with code 0 outside a shutdown" do
|
|
229
|
+
out = expect_restart_delays([0, 1], [1])
|
|
230
|
+
assert_match(/Tube demo\.test\.foo exited with code 0\. Forking it again in 1s\./, out)
|
|
231
|
+
refute_match(/exited normally/, out)
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
it "should start the delays over after a child that ran for MAX_RESTART_DELAY" do
|
|
235
|
+
@worker.stubs(:monotonic_time).returns(0, 0.5, 1, 1.5, 4, 40, 41, 41.5)
|
|
236
|
+
expect_restart_delays([1, 1, 1, 1], [1, 2, 1])
|
|
237
|
+
end
|
|
238
|
+
end # restarting a failed child
|
|
239
|
+
|
|
240
|
+
describe "when beanstalkd is down while children start" do
|
|
241
|
+
before do
|
|
242
|
+
pick_beanstalkd_port
|
|
243
|
+
start_beanstalkd
|
|
244
|
+
@templogger = Templogger.new('/tmp')
|
|
245
|
+
@beanstalk_url = Backburner.configuration.beanstalk_url
|
|
246
|
+
Backburner.configure do |config|
|
|
247
|
+
config.beanstalk_url = "beanstalk://127.0.0.1:#{@port}"
|
|
248
|
+
config.logger = @templogger.logger
|
|
249
|
+
end
|
|
250
|
+
@worker_class.garbage_after = 1
|
|
251
|
+
@watchers = []
|
|
252
|
+
@ignore_forks = true
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
after do
|
|
256
|
+
begin
|
|
257
|
+
@worker_class.shutdown = true
|
|
258
|
+
silenced do
|
|
259
|
+
@worker_class.stop_forks
|
|
260
|
+
Timeout::timeout(2) { sleep 0.1 while @worker_class.child_pids.length > 0 }
|
|
261
|
+
@worker_class.kill_forks
|
|
262
|
+
Timeout::timeout(2) { sleep 0.1 while @worker_class.child_pids.length > 0 }
|
|
263
|
+
end
|
|
264
|
+
ensure
|
|
265
|
+
@watchers.each(&:kill) # once they've reaped the children: one waiting to fork again would wake up in a later test
|
|
266
|
+
stop_beanstalkd
|
|
267
|
+
Backburner.configure do |config|
|
|
268
|
+
config.beanstalk_url = @beanstalk_url
|
|
269
|
+
config.logger = nil
|
|
270
|
+
end
|
|
271
|
+
@templogger.close
|
|
272
|
+
end
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
[1, 2].each do |threads|
|
|
276
|
+
it "should back off instead of forking in a busy loop, and resume when it is back (#{threads} thread(s) per child)" do
|
|
277
|
+
@worker_class.threads_number = threads
|
|
278
|
+
worker = @worker_class.new('foo.bar.outage')
|
|
279
|
+
forks = 0
|
|
280
|
+
watchers = @watchers
|
|
281
|
+
worker.define_singleton_method(:fork_tube) { |name| forks += 1; super(name) }
|
|
282
|
+
worker.define_singleton_method(:create_thread) { |*args, &block| super(*args, &block).tap { |thread| watchers << thread } }
|
|
283
|
+
outage_forks = nil
|
|
284
|
+
silenced(20) do
|
|
285
|
+
worker.start(false)
|
|
286
|
+
sleep 0.5 # let the first child block in reserve
|
|
287
|
+
stop_beanstalkd
|
|
288
|
+
forks_before = forks
|
|
289
|
+
sleep 4
|
|
290
|
+
outage_forks = forks - forks_before
|
|
291
|
+
start_beanstalkd
|
|
292
|
+
@worker_class.enqueue TestJobNoopFork, [], :queue => 'foo.bar.outage'
|
|
293
|
+
wait_for(10) { @templogger.body =~ /Completed TestJobNoopFork/ }
|
|
294
|
+
end
|
|
295
|
+
assert_operator outage_forks, :<=, 4, "forked #{outage_forks} children in 4 s"
|
|
296
|
+
assert_match(/Completed TestJobNoopFork/, @templogger.body)
|
|
297
|
+
end
|
|
298
|
+
end
|
|
299
|
+
end # beanstalkd down
|
|
300
|
+
|
|
165
301
|
describe "fork_inner" do
|
|
166
302
|
|
|
167
303
|
before do
|
|
@@ -193,7 +329,7 @@ describe "Backburner::Workers::ThreadsOnFork module" do
|
|
|
193
329
|
it "should create threads if the number of threads is > 1" do
|
|
194
330
|
worker = @worker_class.new(%(foo))
|
|
195
331
|
@worker_class.expects(:threads_number).returns(5)
|
|
196
|
-
worker.expects(:create_thread).times(5)
|
|
332
|
+
worker.expects(:create_thread).times(5).returns(Thread.new {})
|
|
197
333
|
silenced do
|
|
198
334
|
worker.prepare
|
|
199
335
|
worker.fork_inner('demo.test.foo')
|
|
@@ -204,7 +340,7 @@ describe "Backburner::Workers::ThreadsOnFork module" do
|
|
|
204
340
|
worker = @worker_class.new(%(foo))
|
|
205
341
|
@worker_class.expects(:threads_number).returns(5)
|
|
206
342
|
worker.expects(:run_while_can).times(5)
|
|
207
|
-
def worker.create_thread(*args, &block);
|
|
343
|
+
def worker.create_thread(*args, &block); super(*args, &block).tap(&:join) end
|
|
208
344
|
silenced do
|
|
209
345
|
worker.prepare
|
|
210
346
|
worker.fork_inner('demo.test.foo')
|
|
@@ -218,8 +354,9 @@ describe "Backburner::Workers::ThreadsOnFork module" do
|
|
|
218
354
|
worker = @worker_class.new(%(foo))
|
|
219
355
|
@worker_class.expects(:threads_number).returns(num_threads)
|
|
220
356
|
|
|
357
|
+
fake_conn = Struct.new(:num)
|
|
221
358
|
invocations = Array(1..num_threads).map do |i|
|
|
222
|
-
conn =
|
|
359
|
+
conn = fake_conn.new(i)
|
|
223
360
|
conn.expects(:close)
|
|
224
361
|
conn
|
|
225
362
|
end
|
|
@@ -232,7 +369,7 @@ describe "Backburner::Workers::ThreadsOnFork module" do
|
|
|
232
369
|
worker.expects(:run_while_can).with(conn).when(num_conns.is(conn.num-1)).then(num_conns.is(conn.num))
|
|
233
370
|
end
|
|
234
371
|
|
|
235
|
-
def worker.create_thread(*args, &block);
|
|
372
|
+
def worker.create_thread(*args, &block); super(*args, &block).tap(&:join) end
|
|
236
373
|
silenced do
|
|
237
374
|
worker.prepare
|
|
238
375
|
worker.fork_inner(name)
|
|
@@ -241,6 +378,22 @@ describe "Backburner::Workers::ThreadsOnFork module" do
|
|
|
241
378
|
assert_equal(num_threads, num_conns.current_state)
|
|
242
379
|
end
|
|
243
380
|
|
|
381
|
+
it "should exit once its own threads finish, even while other threads live on" do
|
|
382
|
+
worker = @worker_class.new(%(foo))
|
|
383
|
+
@worker_class.expects(:threads_number).returns(2)
|
|
384
|
+
strays, inner = [], nil
|
|
385
|
+
# Threads a job leaves running, like the one the timeout gem keeps since Ruby 3.2
|
|
386
|
+
worker.define_singleton_method(:run_while_can) { |conn| strays << Thread.new { sleep } }
|
|
387
|
+
silenced do
|
|
388
|
+
worker.prepare
|
|
389
|
+
inner = Thread.new { worker.fork_inner('demo.test.foo') }
|
|
390
|
+
assert inner.join(2), "the child still waited for threads that weren't its own"
|
|
391
|
+
end
|
|
392
|
+
ensure
|
|
393
|
+
inner.kill if inner
|
|
394
|
+
strays.each(&:kill) if strays
|
|
395
|
+
end
|
|
396
|
+
|
|
244
397
|
it "should set @garbage_after, @threads_number and set retries if needed" do
|
|
245
398
|
worker = @worker_class.new(%W(foo1 foo2:10 foo3:20:30 foo4:40:50:60))
|
|
246
399
|
default_threads = 1
|
|
@@ -251,7 +404,7 @@ describe "Backburner::Workers::ThreadsOnFork module" do
|
|
|
251
404
|
@worker_class.any_instance.expects(:coolest_exit).times(3)
|
|
252
405
|
Backburner.configuration.max_job_retries = default_retries
|
|
253
406
|
|
|
254
|
-
worker.expects(:create_thread).times(70)
|
|
407
|
+
worker.expects(:create_thread).times(70).returns(Thread.new {})
|
|
255
408
|
worker.expects(:run_while_can).once
|
|
256
409
|
|
|
257
410
|
silenced do
|
|
@@ -367,6 +520,65 @@ describe "Backburner::Workers::ThreadsOnFork module" do
|
|
|
367
520
|
|
|
368
521
|
end # cleanup on parent
|
|
369
522
|
|
|
523
|
+
describe "a child exiting" do
|
|
524
|
+
before do
|
|
525
|
+
Backburner::Worker.stubs(:gc_disabled_on_exit?).returns(true) # as on Ruby 3.3
|
|
526
|
+
end
|
|
527
|
+
|
|
528
|
+
[:return, :raise, :shutdown].each do |how|
|
|
529
|
+
it "should disable the GC before the interpreter's teardown (#{how})" do
|
|
530
|
+
worker = @worker_class.new(%W(foo))
|
|
531
|
+
gc_disabled = gc_disabled_at_child_exit?(worker, how) do |body|
|
|
532
|
+
worker.wait_for_process(worker.fork_it(&body))
|
|
533
|
+
end
|
|
534
|
+
assert gc_disabled, "the child's GC was still enabled when it exited"
|
|
535
|
+
end
|
|
536
|
+
end
|
|
537
|
+
|
|
538
|
+
it "should leave the GC enabled on a Ruby other than 3.3" do
|
|
539
|
+
Backburner::Worker.stubs(:gc_disabled_on_exit?).returns(false)
|
|
540
|
+
worker = @worker_class.new(%W(foo))
|
|
541
|
+
gc_disabled = gc_disabled_at_child_exit?(worker, :shutdown) do |body|
|
|
542
|
+
worker.wait_for_process(worker.fork_it(&body))
|
|
543
|
+
end
|
|
544
|
+
refute gc_disabled, "the child's GC was disabled when it exited"
|
|
545
|
+
end
|
|
546
|
+
|
|
547
|
+
describe "while running several threads" do
|
|
548
|
+
before do
|
|
549
|
+
@worker_class.threads_number = 2
|
|
550
|
+
@worker_class.garbage_after = nil
|
|
551
|
+
end
|
|
552
|
+
|
|
553
|
+
after do
|
|
554
|
+
@worker_class.threads_number = 1
|
|
555
|
+
@worker_class.garbage_after = 1
|
|
556
|
+
end
|
|
557
|
+
|
|
558
|
+
it "should stop its threads before its at_exit handlers run on a TERM, so they don't work jobs with the GC off" do
|
|
559
|
+
worker = @worker_class.new(%W(foo))
|
|
560
|
+
reader, writer = IO.pipe
|
|
561
|
+
pid = worker.fork_it do
|
|
562
|
+
reader.close
|
|
563
|
+
$stdout.reopen(File::NULL)
|
|
564
|
+
$stderr.reopen(File::NULL)
|
|
565
|
+
$jobs_worked = 0
|
|
566
|
+
worker.define_singleton_method(:run_while_can) { |*| loop { $jobs_worked += 1; sleep 0.01 } } # jobs keep coming
|
|
567
|
+
# An app's slow at_exit handler (an error reporter flushing, say)
|
|
568
|
+
at_exit { sleep 0.05; before = $jobs_worked; sleep 0.3; writer.write(($jobs_worked - before).to_s); writer.close }
|
|
569
|
+
Thread.new { sleep 0.3; Process.kill(:TERM, Process.pid) }
|
|
570
|
+
worker.fork_inner("demo.test.foo")
|
|
571
|
+
end
|
|
572
|
+
writer.close
|
|
573
|
+
worker.wait_for_process(pid)
|
|
574
|
+
assert_equal "0", reader.read, "the child's threads kept working jobs while its at_exit handlers ran"
|
|
575
|
+
ensure
|
|
576
|
+
reader.close if reader && !reader.closed?
|
|
577
|
+
writer.close if writer && !writer.closed?
|
|
578
|
+
end
|
|
579
|
+
end
|
|
580
|
+
end
|
|
581
|
+
|
|
370
582
|
describe "practical tests" do
|
|
371
583
|
|
|
372
584
|
before do
|
|
@@ -376,7 +588,7 @@ describe "Backburner::Workers::ThreadsOnFork module" do
|
|
|
376
588
|
$worker_success = false
|
|
377
589
|
$worker_raise = false
|
|
378
590
|
clear_jobs!('response')
|
|
379
|
-
clear_jobs!('foo.bar.1', 'foo.bar.2', 'foo.bar.3', 'foo.bar.4', 'foo.bar.5', 'foo.bar.6', 'foo.bar.7')
|
|
591
|
+
clear_jobs!('foo.bar.1', 'foo.bar.2', 'foo.bar.3', 'foo.bar.4', 'foo.bar.5', 'foo.bar.6', 'foo.bar.7', 'foo.bar.8')
|
|
380
592
|
@worker_class.threads_number = 1
|
|
381
593
|
@worker_class.garbage_after = 10
|
|
382
594
|
|
|
@@ -391,7 +603,7 @@ describe "Backburner::Workers::ThreadsOnFork module" do
|
|
|
391
603
|
after do
|
|
392
604
|
@templogger.close
|
|
393
605
|
clear_jobs!('response')
|
|
394
|
-
clear_jobs!('foo.bar.1', 'foo.bar.2', 'foo.bar.3', 'foo.bar.4', 'foo.bar.5', 'foo.bar.6', 'foo.bar.7')
|
|
606
|
+
clear_jobs!('foo.bar.1', 'foo.bar.2', 'foo.bar.3', 'foo.bar.4', 'foo.bar.5', 'foo.bar.6', 'foo.bar.7', 'foo.bar.8')
|
|
395
607
|
@worker_class.threads_number = 1
|
|
396
608
|
@worker_class.shutdown = true
|
|
397
609
|
silenced do
|
|
@@ -479,6 +691,61 @@ describe "Backburner::Workers::ThreadsOnFork module" do
|
|
|
479
691
|
end
|
|
480
692
|
assert_equal num_jobs, $worker_test_count
|
|
481
693
|
end # multithreaded
|
|
694
|
+
|
|
695
|
+
it "should give each forked tube its own connection when running one thread per tube" do
|
|
696
|
+
num_jobs = 5
|
|
697
|
+
tubes = %w(foo.bar.7 foo.bar.8)
|
|
698
|
+
@worker = @worker_class.new(tubes)
|
|
699
|
+
@worker.start(false)
|
|
700
|
+
tubes.each do |tube|
|
|
701
|
+
num_jobs.times { @worker_class.enqueue TestJobNoopFork, [], :queue => tube }
|
|
702
|
+
end
|
|
703
|
+
|
|
704
|
+
completed = lambda { @templogger.body.scan(/Completed TestJobNoopFork/).size }
|
|
705
|
+
deadline = Time.now + 10
|
|
706
|
+
sleep 0.1 until completed.call >= tubes.size * num_jobs || Time.now > deadline
|
|
707
|
+
|
|
708
|
+
assert_equal tubes.size * num_jobs, completed.call, @templogger.body
|
|
709
|
+
conn = beanstalk_connection
|
|
710
|
+
tubes.each do |tube|
|
|
711
|
+
stats = conn.tubes["demo.test.#{tube}"].stats
|
|
712
|
+
assert_equal 0, stats.current_jobs_ready + stats.current_jobs_reserved, "jobs left in #{tube}"
|
|
713
|
+
end
|
|
714
|
+
conn.close
|
|
715
|
+
end # one thread per tube
|
|
716
|
+
|
|
717
|
+
it "should release a job whose class can't be found and keep its child working" do
|
|
718
|
+
clear_jobs!('foo.bar.9')
|
|
719
|
+
conn = beanstalk_connection
|
|
720
|
+
conn.tubes['demo.test.foo.bar.9'].put(Backburner.configuration.job_serializer_proc.call(:class => 'TestRemovedJobFork', :args => []), :ttr => 30)
|
|
721
|
+
@worker = @worker_class.new('foo.bar.9')
|
|
722
|
+
@worker.start(false)
|
|
723
|
+
@worker_class.enqueue TestJobNoopFork, [], :queue => 'foo.bar.9'
|
|
724
|
+
|
|
725
|
+
silenced(5) { @templogger.wait_for_match(/Completed TestJobNoopFork/m) }
|
|
726
|
+
assert_match(/TestRemovedJobFork: class not found; released for 30s/, @templogger.body)
|
|
727
|
+
refute_match(/Catastrophic failure/, @templogger.body)
|
|
728
|
+
stats = conn.tubes['demo.test.foo.bar.9'].stats
|
|
729
|
+
assert_equal [0, 1], [stats.current_jobs_buried, stats.current_jobs_delayed]
|
|
730
|
+
ensure
|
|
731
|
+
conn.close if conn
|
|
732
|
+
clear_jobs!('foo.bar.9')
|
|
733
|
+
end # unknown class
|
|
734
|
+
|
|
735
|
+
it "should recycle a child running several threads after garbage_after jobs and keep working" do
|
|
736
|
+
clear_jobs!('foo.bar.10')
|
|
737
|
+
@worker_class.threads_number = 2
|
|
738
|
+
@worker_class.garbage_after = 2
|
|
739
|
+
@worker = @worker_class.new('foo.bar.10')
|
|
740
|
+
@worker.start(false)
|
|
741
|
+
6.times { @worker_class.enqueue TestJobNoopFork, [], :queue => 'foo.bar.10' }
|
|
742
|
+
|
|
743
|
+
completed = lambda { @templogger.body.scan(/Completed TestJobNoopFork/).size }
|
|
744
|
+
wait_for(10) { completed.call >= 6 }
|
|
745
|
+
assert_equal 6, completed.call, @templogger.body
|
|
746
|
+
ensure
|
|
747
|
+
clear_jobs!('foo.bar.10')
|
|
748
|
+
end # recycled multithreaded child
|
|
482
749
|
end # practical tests
|
|
483
750
|
end # forking and threading
|
|
484
751
|
end # Backburner::Workers::ThreadsOnFork module
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: backburner
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.8.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Nathan Esquenazi
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2026-11-01 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: beaneater
|
|
@@ -58,6 +58,20 @@ dependencies:
|
|
|
58
58
|
- - ">="
|
|
59
59
|
- !ruby/object:Gem::Version
|
|
60
60
|
version: 1.0.1
|
|
61
|
+
- !ruby/object:Gem::Dependency
|
|
62
|
+
name: logger
|
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
|
64
|
+
requirements:
|
|
65
|
+
- - ">="
|
|
66
|
+
- !ruby/object:Gem::Version
|
|
67
|
+
version: '0'
|
|
68
|
+
type: :runtime
|
|
69
|
+
prerelease: false
|
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
71
|
+
requirements:
|
|
72
|
+
- - ">="
|
|
73
|
+
- !ruby/object:Gem::Version
|
|
74
|
+
version: '0'
|
|
61
75
|
- !ruby/object:Gem::Dependency
|
|
62
76
|
name: rake
|
|
63
77
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -78,14 +92,28 @@ dependencies:
|
|
|
78
92
|
requirements:
|
|
79
93
|
- - '='
|
|
80
94
|
- !ruby/object:Gem::Version
|
|
81
|
-
version:
|
|
95
|
+
version: 5.1.0
|
|
82
96
|
type: :development
|
|
83
97
|
prerelease: false
|
|
84
98
|
version_requirements: !ruby/object:Gem::Requirement
|
|
85
99
|
requirements:
|
|
86
100
|
- - '='
|
|
87
101
|
- !ruby/object:Gem::Version
|
|
88
|
-
version:
|
|
102
|
+
version: 5.1.0
|
|
103
|
+
- !ruby/object:Gem::Dependency
|
|
104
|
+
name: mutex_m
|
|
105
|
+
requirement: !ruby/object:Gem::Requirement
|
|
106
|
+
requirements:
|
|
107
|
+
- - ">="
|
|
108
|
+
- !ruby/object:Gem::Version
|
|
109
|
+
version: '0'
|
|
110
|
+
type: :development
|
|
111
|
+
prerelease: false
|
|
112
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
113
|
+
requirements:
|
|
114
|
+
- - ">="
|
|
115
|
+
- !ruby/object:Gem::Version
|
|
116
|
+
version: '0'
|
|
89
117
|
- !ruby/object:Gem::Dependency
|
|
90
118
|
name: mocha
|
|
91
119
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -100,6 +128,34 @@ dependencies:
|
|
|
100
128
|
- - ">="
|
|
101
129
|
- !ruby/object:Gem::Version
|
|
102
130
|
version: '0'
|
|
131
|
+
- !ruby/object:Gem::Dependency
|
|
132
|
+
name: activejob
|
|
133
|
+
requirement: !ruby/object:Gem::Requirement
|
|
134
|
+
requirements:
|
|
135
|
+
- - ">="
|
|
136
|
+
- !ruby/object:Gem::Version
|
|
137
|
+
version: '6.0'
|
|
138
|
+
type: :development
|
|
139
|
+
prerelease: false
|
|
140
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
141
|
+
requirements:
|
|
142
|
+
- - ">="
|
|
143
|
+
- !ruby/object:Gem::Version
|
|
144
|
+
version: '6.0'
|
|
145
|
+
- !ruby/object:Gem::Dependency
|
|
146
|
+
name: activesupport
|
|
147
|
+
requirement: !ruby/object:Gem::Requirement
|
|
148
|
+
requirements:
|
|
149
|
+
- - ">="
|
|
150
|
+
- !ruby/object:Gem::Version
|
|
151
|
+
version: '6.0'
|
|
152
|
+
type: :development
|
|
153
|
+
prerelease: false
|
|
154
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
155
|
+
requirements:
|
|
156
|
+
- - ">="
|
|
157
|
+
- !ruby/object:Gem::Version
|
|
158
|
+
version: '6.0'
|
|
103
159
|
description: Beanstalk background job processing made easy
|
|
104
160
|
email:
|
|
105
161
|
- nesquena@gmail.com
|
|
@@ -108,8 +164,10 @@ executables:
|
|
|
108
164
|
extensions: []
|
|
109
165
|
extra_rdoc_files: []
|
|
110
166
|
files:
|
|
167
|
+
- ".github/workflows/ci.yml"
|
|
168
|
+
- ".github/workflows/github-release.yml"
|
|
169
|
+
- ".github/workflows/release.yml"
|
|
111
170
|
- ".gitignore"
|
|
112
|
-
- ".travis.yml"
|
|
113
171
|
- CHANGELOG.md
|
|
114
172
|
- CONTRIBUTING.md
|
|
115
173
|
- Gemfile
|
|
@@ -120,7 +178,6 @@ files:
|
|
|
120
178
|
- TODO
|
|
121
179
|
- backburner.gemspec
|
|
122
180
|
- bin/backburner
|
|
123
|
-
- circle.yml
|
|
124
181
|
- examples/custom.rb
|
|
125
182
|
- examples/demo.rb
|
|
126
183
|
- examples/god.rb
|
|
@@ -128,11 +185,13 @@ files:
|
|
|
128
185
|
- examples/retried.rb
|
|
129
186
|
- examples/simple.rb
|
|
130
187
|
- examples/stress.rb
|
|
188
|
+
- lib/active_job/queue_adapters/backburner_adapter.rb
|
|
131
189
|
- lib/backburner.rb
|
|
132
190
|
- lib/backburner/async_proxy.rb
|
|
133
191
|
- lib/backburner/cli.rb
|
|
134
192
|
- lib/backburner/configuration.rb
|
|
135
193
|
- lib/backburner/connection.rb
|
|
194
|
+
- lib/backburner/enqueue_pool.rb
|
|
136
195
|
- lib/backburner/helpers.rb
|
|
137
196
|
- lib/backburner/hooks.rb
|
|
138
197
|
- lib/backburner/job.rb
|
|
@@ -146,9 +205,19 @@ files:
|
|
|
146
205
|
- lib/backburner/workers/simple.rb
|
|
147
206
|
- lib/backburner/workers/threading.rb
|
|
148
207
|
- lib/backburner/workers/threads_on_fork.rb
|
|
208
|
+
- test/active_job_adapter_test.rb
|
|
149
209
|
- test/async_proxy_test.rb
|
|
150
210
|
- test/back_burner_test.rb
|
|
151
211
|
- test/connection_test.rb
|
|
212
|
+
- test/fixtures/active_job_app.rb
|
|
213
|
+
- test/fixtures/active_jobs.rb
|
|
214
|
+
- test/fixtures/autoload/test_broken_name_job.rb
|
|
215
|
+
- test/fixtures/autoload/test_broken_no_method_job.rb
|
|
216
|
+
- test/fixtures/autoload/test_broken_require_job.rb
|
|
217
|
+
- test/fixtures/autoload/test_broken_runtime_job.rb
|
|
218
|
+
- test/fixtures/autoload/test_broken_syntax_job.rb
|
|
219
|
+
- test/fixtures/autoload/test_broken_timeout_job.rb
|
|
220
|
+
- test/fixtures/autoload/test_same_name_job.rb
|
|
152
221
|
- test/fixtures/hooked.rb
|
|
153
222
|
- test/fixtures/test_fork_jobs.rb
|
|
154
223
|
- test/fixtures/test_forking_jobs.rb
|
|
@@ -179,21 +248,31 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
179
248
|
requirements:
|
|
180
249
|
- - ">="
|
|
181
250
|
- !ruby/object:Gem::Version
|
|
182
|
-
version: '
|
|
251
|
+
version: '2.7'
|
|
183
252
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
184
253
|
requirements:
|
|
185
254
|
- - ">="
|
|
186
255
|
- !ruby/object:Gem::Version
|
|
187
256
|
version: '0'
|
|
188
257
|
requirements: []
|
|
189
|
-
rubygems_version: 3.
|
|
258
|
+
rubygems_version: 3.5.22
|
|
190
259
|
signing_key:
|
|
191
260
|
specification_version: 4
|
|
192
261
|
summary: Reliable beanstalk background job processing made easy for Ruby and Sinatra
|
|
193
262
|
test_files:
|
|
263
|
+
- test/active_job_adapter_test.rb
|
|
194
264
|
- test/async_proxy_test.rb
|
|
195
265
|
- test/back_burner_test.rb
|
|
196
266
|
- test/connection_test.rb
|
|
267
|
+
- test/fixtures/active_job_app.rb
|
|
268
|
+
- test/fixtures/active_jobs.rb
|
|
269
|
+
- test/fixtures/autoload/test_broken_name_job.rb
|
|
270
|
+
- test/fixtures/autoload/test_broken_no_method_job.rb
|
|
271
|
+
- test/fixtures/autoload/test_broken_require_job.rb
|
|
272
|
+
- test/fixtures/autoload/test_broken_runtime_job.rb
|
|
273
|
+
- test/fixtures/autoload/test_broken_syntax_job.rb
|
|
274
|
+
- test/fixtures/autoload/test_broken_timeout_job.rb
|
|
275
|
+
- test/fixtures/autoload/test_same_name_job.rb
|
|
197
276
|
- test/fixtures/hooked.rb
|
|
198
277
|
- test/fixtures/test_fork_jobs.rb
|
|
199
278
|
- test/fixtures/test_forking_jobs.rb
|
data/.travis.yml
DELETED
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
# http://about.travis-ci.org/docs/user/build-configuration/
|
|
2
|
-
rvm:
|
|
3
|
-
- 1.9.3
|
|
4
|
-
- 2.0.0
|
|
5
|
-
- 2.1
|
|
6
|
-
- 2.2
|
|
7
|
-
- 2.3
|
|
8
|
-
- 2.4
|
|
9
|
-
- 2.5
|
|
10
|
-
- rbx-2
|
|
11
|
-
before_install:
|
|
12
|
-
- curl -L https://github.com/kr/beanstalkd/archive/v1.9.tar.gz | tar xz -C /tmp
|
|
13
|
-
- cd /tmp/beanstalkd-1.9/
|
|
14
|
-
- make
|
|
15
|
-
- ./beanstalkd &
|
|
16
|
-
- cd $TRAVIS_BUILD_DIR
|
|
17
|
-
- gem update --system
|
|
18
|
-
- gem update bundler
|
|
19
|
-
matrix:
|
|
20
|
-
allow_failures:
|
|
21
|
-
- rvm: rbx-2
|
|
22
|
-
- rvm: 2.0.0
|
|
23
|
-
script:
|
|
24
|
-
- bundle exec rake test
|
|
25
|
-
gemfile: Gemfile
|
|
26
|
-
notifications:
|
|
27
|
-
recipients:
|
|
28
|
-
- nesquena@gmail.com
|
|
29
|
-
- therealdave.myron@gmail.com
|
data/circle.yml
DELETED