backburner-allq 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/.travis.yml +29 -0
- data/CHANGELOG.md +133 -0
- data/CONTRIBUTING.md +37 -0
- data/Gemfile +4 -0
- data/HOOKS.md +99 -0
- data/LICENSE +22 -0
- data/README.md +658 -0
- data/Rakefile +17 -0
- data/TODO +4 -0
- data/backburner-allq.gemspec +26 -0
- data/bin/backburner +7 -0
- data/circle.yml +3 -0
- data/deploy.sh +3 -0
- data/examples/custom.rb +25 -0
- data/examples/demo.rb +60 -0
- data/examples/god.rb +46 -0
- data/examples/hooked.rb +87 -0
- data/examples/retried.rb +31 -0
- data/examples/simple.rb +43 -0
- data/examples/stress.rb +31 -0
- data/lib/backburner.rb +75 -0
- data/lib/backburner/allq_wrapper.rb +317 -0
- data/lib/backburner/async_proxy.rb +25 -0
- data/lib/backburner/cli.rb +53 -0
- data/lib/backburner/configuration.rb +48 -0
- data/lib/backburner/connection.rb +157 -0
- data/lib/backburner/helpers.rb +193 -0
- data/lib/backburner/hooks.rb +53 -0
- data/lib/backburner/job.rb +118 -0
- data/lib/backburner/logger.rb +53 -0
- data/lib/backburner/performable.rb +95 -0
- data/lib/backburner/queue.rb +145 -0
- data/lib/backburner/tasks.rb +54 -0
- data/lib/backburner/version.rb +3 -0
- data/lib/backburner/worker.rb +221 -0
- data/lib/backburner/workers/forking.rb +52 -0
- data/lib/backburner/workers/simple.rb +29 -0
- data/lib/backburner/workers/threading.rb +163 -0
- data/lib/backburner/workers/threads_on_fork.rb +263 -0
- data/test/async_proxy_test.rb +36 -0
- data/test/back_burner_test.rb +88 -0
- data/test/connection_test.rb +179 -0
- data/test/fixtures/hooked.rb +122 -0
- data/test/fixtures/test_fork_jobs.rb +72 -0
- data/test/fixtures/test_forking_jobs.rb +56 -0
- data/test/fixtures/test_jobs.rb +87 -0
- data/test/fixtures/test_queue_settings.rb +14 -0
- data/test/helpers/templogger.rb +22 -0
- data/test/helpers_test.rb +278 -0
- data/test/hooks_test.rb +112 -0
- data/test/job_test.rb +185 -0
- data/test/logger_test.rb +44 -0
- data/test/performable_test.rb +88 -0
- data/test/queue_test.rb +69 -0
- data/test/test_helper.rb +128 -0
- data/test/worker_test.rb +157 -0
- data/test/workers/forking_worker_test.rb +181 -0
- data/test/workers/simple_worker_test.rb +350 -0
- data/test/workers/threading_worker_test.rb +104 -0
- data/test/workers/threads_on_fork_worker_test.rb +484 -0
- metadata +217 -0
@@ -0,0 +1,484 @@
|
|
1
|
+
require File.expand_path('../../test_helper', __FILE__)
|
2
|
+
require File.expand_path('../../fixtures/test_fork_jobs', __FILE__)
|
3
|
+
require File.expand_path('../../fixtures/test_queue_settings', __FILE__)
|
4
|
+
|
5
|
+
describe "Backburner::Workers::ThreadsOnFork module" do
|
6
|
+
|
7
|
+
before do
|
8
|
+
Backburner.default_queues.clear
|
9
|
+
@worker_class = Backburner::Workers::ThreadsOnFork
|
10
|
+
@worker_class.shutdown = false
|
11
|
+
@worker_class.is_child = false
|
12
|
+
@worker_class.threads_number = 1
|
13
|
+
@worker_class.garbage_after = 1
|
14
|
+
@ignore_forks = false
|
15
|
+
end
|
16
|
+
|
17
|
+
after do
|
18
|
+
Backburner.configure { |config| config.max_job_retries = 0; config.retry_delay = 5; config.logger = nil }
|
19
|
+
unless @ignore_forks
|
20
|
+
cpids = @worker_class.instance_variable_get("@child_pids")
|
21
|
+
if cpids && cpids.length > 0
|
22
|
+
raise "Why is there forks alive?"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
describe "for process_tube_names method" do
|
28
|
+
it "should interpreter the job_name:threads_limit:garbage_after:retries format" do
|
29
|
+
worker = @worker_class.new(["foo:1:2:3"])
|
30
|
+
assert_equal ["foo"], worker.tube_names
|
31
|
+
end
|
32
|
+
|
33
|
+
it "should interpreter event if is missing values" do
|
34
|
+
tubes = %W(foo1:1:2:3 foo2:4:5 foo3:6 foo4 foo5::7:8 foo6:::9 foo7::10)
|
35
|
+
worker = @worker_class.new(tubes)
|
36
|
+
assert_equal %W(foo1 foo2 foo3 foo4 foo5 foo6 foo7), worker.tube_names
|
37
|
+
end
|
38
|
+
|
39
|
+
it "should store interpreted values correctly" do
|
40
|
+
tubes = %W(foo1:1:2:3 foo2:4:5 foo3:6 foo4 foo5::7:8 foo6:::9 foo7::10)
|
41
|
+
worker = @worker_class.new(tubes)
|
42
|
+
assert_equal({
|
43
|
+
"demo.test.foo1" => { :threads => 1, :garbage => 2, :retries => 3 },
|
44
|
+
"demo.test.foo2" => { :threads => 4, :garbage => 5, :retries => nil },
|
45
|
+
"demo.test.foo3" => { :threads => 6, :garbage => nil, :retries => nil },
|
46
|
+
"demo.test.foo4" => { :threads => nil, :garbage => nil, :retries => nil },
|
47
|
+
"demo.test.foo5" => { :threads => nil, :garbage => 7, :retries => 8 },
|
48
|
+
"demo.test.foo6" => { :threads => nil, :garbage => nil, :retries => 9 },
|
49
|
+
"demo.test.foo7" => { :threads => nil, :garbage => 10, :retries => nil }
|
50
|
+
}, worker.instance_variable_get("@tubes_data"))
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
describe "for process_tube_settings" do
|
55
|
+
it "should set the settings specified by queue name in class" do
|
56
|
+
worker = @worker_class.new
|
57
|
+
assert_equal(worker.instance_variable_get("@tubes_data")['demo.test.job-settings'], { :threads => 5, :garbage => 10, :retries => 6 })
|
58
|
+
end
|
59
|
+
|
60
|
+
it 'should override the tube settings if they are specified directly at class level' do
|
61
|
+
worker = @worker_class.new
|
62
|
+
assert_equal(worker.instance_variable_get("@tubes_data")['demo.test.job-settings-override'], { :threads => 10, :garbage => 1000, :retries => 2 })
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
describe "for prepare method" do
|
67
|
+
before do
|
68
|
+
Backburner.configure { |config| config.logger = false }
|
69
|
+
end
|
70
|
+
|
71
|
+
it "should make tube names array always unique to avoid duplication" do
|
72
|
+
worker = @worker_class.new(["foo", "demo.test.foo"])
|
73
|
+
capture_stdout { worker.prepare }
|
74
|
+
assert_equal ["demo.test.foo"], worker.tube_names
|
75
|
+
end
|
76
|
+
|
77
|
+
it "should watch specified tubes" do
|
78
|
+
worker = @worker_class.new(["foo", "bar"])
|
79
|
+
out = capture_stdout { worker.prepare }
|
80
|
+
assert_equal ["demo.test.foo", "demo.test.bar"], worker.tube_names
|
81
|
+
assert_match(/demo\.test\.foo/, out)
|
82
|
+
end # multiple
|
83
|
+
|
84
|
+
it "should watch single tube" do
|
85
|
+
worker = @worker_class.new("foo")
|
86
|
+
out = capture_stdout { worker.prepare }
|
87
|
+
assert_equal ["demo.test.foo"], worker.tube_names
|
88
|
+
assert_match(/demo\.test\.foo/, out)
|
89
|
+
end # single
|
90
|
+
|
91
|
+
it "should respect default_queues settings" do
|
92
|
+
Backburner.default_queues.concat(["foo", "bar"])
|
93
|
+
worker = @worker_class.new
|
94
|
+
out = capture_stdout { worker.prepare }
|
95
|
+
assert_equal ["demo.test.foo", "demo.test.bar"], worker.tube_names
|
96
|
+
assert_match(/demo\.test\.foo/, out)
|
97
|
+
end
|
98
|
+
|
99
|
+
it "should assign based on all tubes" do
|
100
|
+
@worker_class.any_instance.expects(:all_existing_queues).once.returns("bar")
|
101
|
+
worker = @worker_class.new
|
102
|
+
out = capture_stdout { worker.prepare }
|
103
|
+
assert_equal ["demo.test.bar"], worker.tube_names
|
104
|
+
assert_match(/demo\.test\.bar/, out)
|
105
|
+
end # all assign
|
106
|
+
|
107
|
+
it "should properly retrieve all tubes" do
|
108
|
+
worker = @worker_class.new
|
109
|
+
out = capture_stdout { worker.prepare }
|
110
|
+
assert_contains worker.tube_names, "demo.test.test-job-fork"
|
111
|
+
assert_match(/demo\.test\.test-job-fork/, out)
|
112
|
+
end # all read
|
113
|
+
end # prepare
|
114
|
+
|
115
|
+
describe "forking and threading" do
|
116
|
+
|
117
|
+
it "start should call fork_and_watch for each tube" do
|
118
|
+
worker = @worker_class.new(%W(foo bar))
|
119
|
+
worker.expects(:fork_and_watch).with("demo.test.foo").once
|
120
|
+
worker.expects(:fork_and_watch).with("demo.test.bar").once
|
121
|
+
silenced { worker.start(false) }
|
122
|
+
end
|
123
|
+
|
124
|
+
it "fork_and_watch should create a thread to fork and watch" do
|
125
|
+
worker = @worker_class.new(%(foo))
|
126
|
+
worker.expects(:create_thread).once.with("demo.test.foo")
|
127
|
+
silenced { worker.start(false) }
|
128
|
+
end
|
129
|
+
|
130
|
+
it "fork_and_watch thread should wait with wait_for_process" do
|
131
|
+
process_exit = stub('process_exit')
|
132
|
+
process_exit.expects(:exitstatus).returns(99)
|
133
|
+
worker = @worker_class.new(%(foo))
|
134
|
+
worker.expects(:wait_for_process).with(12).returns([nil, process_exit])
|
135
|
+
|
136
|
+
wc = @worker_class
|
137
|
+
# TODO: Is there a best way do do this?
|
138
|
+
worker.define_singleton_method :fork_it do
|
139
|
+
wc.shutdown = true
|
140
|
+
12
|
141
|
+
end
|
142
|
+
def worker.create_thread(*args, &block); block.call(*args) end
|
143
|
+
|
144
|
+
out = silenced(2) { worker.start(false) }
|
145
|
+
refute_match(/Catastrophic failure/, out)
|
146
|
+
end
|
147
|
+
|
148
|
+
it "fork_and_watch thread should log an error if exitstatus is != 99" do
|
149
|
+
process_exit = stub('process_exit')
|
150
|
+
process_exit.expects(:exitstatus).twice.returns(0)
|
151
|
+
worker = @worker_class.new(%(foo))
|
152
|
+
worker.expects(:wait_for_process).with(12).returns([nil, process_exit])
|
153
|
+
|
154
|
+
wc = @worker_class
|
155
|
+
# TODO: Is there a best way do do this?
|
156
|
+
worker.define_singleton_method :fork_it do
|
157
|
+
wc.shutdown = true
|
158
|
+
12
|
159
|
+
end
|
160
|
+
def worker.create_thread(*args, &block); block.call(*args) end
|
161
|
+
out = silenced(2) { worker.start(false) }
|
162
|
+
assert_match(/Catastrophic failure: tube demo\.test\.foo exited with code 0\./, out)
|
163
|
+
end
|
164
|
+
|
165
|
+
describe "fork_inner" do
|
166
|
+
|
167
|
+
before do
|
168
|
+
@worker_class.any_instance.expects(:coolest_exit).once
|
169
|
+
end
|
170
|
+
|
171
|
+
it "should watch just the channel it receive as argument" do
|
172
|
+
worker = @worker_class.new(%(foo))
|
173
|
+
@worker_class.expects(:threads_number).returns(1)
|
174
|
+
worker.expects(:run_while_can).once
|
175
|
+
silenced do
|
176
|
+
worker.prepare
|
177
|
+
worker.fork_inner('demo.test.bar')
|
178
|
+
end
|
179
|
+
assert_same_elements %W(demo.test.bar), worker.connection.tubes.watched.map(&:name)
|
180
|
+
end
|
181
|
+
|
182
|
+
it "should not create threads if the number of threads is 1" do
|
183
|
+
worker = @worker_class.new(%(foo))
|
184
|
+
@worker_class.expects(:threads_number).returns(1)
|
185
|
+
worker.expects(:run_while_can).once
|
186
|
+
worker.expects(:create_thread).never
|
187
|
+
silenced do
|
188
|
+
worker.prepare
|
189
|
+
worker.fork_inner('demo.test.foo')
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
it "should create threads if the number of threads is > 1" do
|
194
|
+
worker = @worker_class.new(%(foo))
|
195
|
+
@worker_class.expects(:threads_number).returns(5)
|
196
|
+
worker.expects(:create_thread).times(5)
|
197
|
+
silenced do
|
198
|
+
worker.prepare
|
199
|
+
worker.fork_inner('demo.test.foo')
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
203
|
+
it "should create threads that call run_while_can" do
|
204
|
+
worker = @worker_class.new(%(foo))
|
205
|
+
@worker_class.expects(:threads_number).returns(5)
|
206
|
+
worker.expects(:run_while_can).times(5)
|
207
|
+
def worker.create_thread(*args, &block); block.call(*args) end
|
208
|
+
silenced do
|
209
|
+
worker.prepare
|
210
|
+
worker.fork_inner('demo.test.foo')
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
it "should create a connection for each thread" do
|
215
|
+
name = 'demo.test.foo'
|
216
|
+
num_threads = 3
|
217
|
+
|
218
|
+
worker = @worker_class.new(%(foo))
|
219
|
+
@worker_class.expects(:threads_number).returns(num_threads)
|
220
|
+
|
221
|
+
invocations = Array(1..num_threads).map do |i|
|
222
|
+
conn = OpenStruct.new(:num => i)
|
223
|
+
conn.expects(:close)
|
224
|
+
conn
|
225
|
+
end
|
226
|
+
Backburner::Connection.expects(:new).times(num_threads).returns(*invocations)
|
227
|
+
|
228
|
+
# ensure each invocation of run_while_can is with a different connection
|
229
|
+
num_conns = states('num_conns').starts_as(0)
|
230
|
+
invocations.each do |conn|
|
231
|
+
worker.expects(:watch_tube).with(name, conn)
|
232
|
+
worker.expects(:run_while_can).with(conn).when(num_conns.is(conn.num-1)).then(num_conns.is(conn.num))
|
233
|
+
end
|
234
|
+
|
235
|
+
def worker.create_thread(*args, &block); block.call(*args) end
|
236
|
+
silenced do
|
237
|
+
worker.prepare
|
238
|
+
worker.fork_inner(name)
|
239
|
+
end
|
240
|
+
|
241
|
+
assert_equal(num_threads, num_conns.current_state)
|
242
|
+
end
|
243
|
+
|
244
|
+
it "should set @garbage_after, @threads_number and set retries if needed" do
|
245
|
+
worker = @worker_class.new(%W(foo1 foo2:10 foo3:20:30 foo4:40:50:60))
|
246
|
+
default_threads = 1
|
247
|
+
default_garbage = 5
|
248
|
+
default_retries = 100
|
249
|
+
@worker_class.expects(:threads_number).times(1).returns(default_threads)
|
250
|
+
@worker_class.expects(:garbage_after).times(2).returns(default_garbage)
|
251
|
+
@worker_class.any_instance.expects(:coolest_exit).times(3)
|
252
|
+
Backburner.configuration.max_job_retries = default_retries
|
253
|
+
|
254
|
+
worker.expects(:create_thread).times(70)
|
255
|
+
worker.expects(:run_while_can).once
|
256
|
+
|
257
|
+
silenced do
|
258
|
+
worker.prepare
|
259
|
+
worker.fork_inner('demo.test.foo1')
|
260
|
+
end
|
261
|
+
|
262
|
+
assert_equal worker.instance_variable_get("@threads_number"), default_threads
|
263
|
+
assert_equal worker.instance_variable_get("@garbage_after"), default_garbage
|
264
|
+
assert_equal Backburner.configuration.max_job_retries, default_retries
|
265
|
+
|
266
|
+
silenced do
|
267
|
+
worker.fork_inner('demo.test.foo2')
|
268
|
+
end
|
269
|
+
|
270
|
+
assert_equal worker.instance_variable_get("@threads_number"), 10
|
271
|
+
assert_equal worker.instance_variable_get("@garbage_after"), default_garbage
|
272
|
+
assert_equal Backburner.configuration.max_job_retries, default_retries
|
273
|
+
|
274
|
+
silenced do
|
275
|
+
worker.fork_inner('demo.test.foo3')
|
276
|
+
end
|
277
|
+
|
278
|
+
assert_equal worker.instance_variable_get("@threads_number"), 20
|
279
|
+
assert_equal worker.instance_variable_get("@garbage_after"), 30
|
280
|
+
assert_equal Backburner.configuration.max_job_retries, default_retries
|
281
|
+
|
282
|
+
silenced do
|
283
|
+
worker.fork_inner('demo.test.foo4')
|
284
|
+
end
|
285
|
+
|
286
|
+
assert_equal worker.instance_variable_get("@threads_number"), 40
|
287
|
+
assert_equal worker.instance_variable_get("@garbage_after"), 50
|
288
|
+
assert_equal Backburner.configuration.max_job_retries, 60
|
289
|
+
end
|
290
|
+
|
291
|
+
end
|
292
|
+
|
293
|
+
describe "cleanup on parent" do
|
294
|
+
|
295
|
+
it "child_pids should return a list of alive children pids" do
|
296
|
+
worker = @worker_class.new(%W(foo))
|
297
|
+
Kernel.expects(:fork).once.returns(12345)
|
298
|
+
Process.expects(:kill).with(0, 12345).once
|
299
|
+
Process.expects(:pid).once.returns(12346)
|
300
|
+
assert_equal [], @worker_class.child_pids
|
301
|
+
worker.fork_it {}
|
302
|
+
child_pids = @worker_class.child_pids
|
303
|
+
assert_equal [12345], child_pids
|
304
|
+
child_pids.clear
|
305
|
+
end
|
306
|
+
|
307
|
+
it "child_pids should return an empty array if is_child" do
|
308
|
+
Process.expects(:pid).never
|
309
|
+
@worker_class.is_child = true
|
310
|
+
@worker_class.child_pids << 12345
|
311
|
+
assert_equal [], @worker_class.child_pids
|
312
|
+
end
|
313
|
+
|
314
|
+
it "stop_forks should send a SIGTERM for every child" do
|
315
|
+
Process.expects(:pid).returns(12346).at_least(1)
|
316
|
+
Process.expects(:kill).with(0, 12345).at_least(1)
|
317
|
+
Process.expects(:kill).with(0, 12347).at_least(1)
|
318
|
+
Process.expects(:kill).with("SIGTERM", 12345)
|
319
|
+
Process.expects(:kill).with("SIGTERM", 12347)
|
320
|
+
@worker_class.child_pids << 12345
|
321
|
+
@worker_class.child_pids << 12347
|
322
|
+
assert_equal [12345, 12347], @worker_class.child_pids
|
323
|
+
@worker_class.stop_forks
|
324
|
+
@worker_class.child_pids.clear
|
325
|
+
end
|
326
|
+
|
327
|
+
it "kill_forks should send a SIGKILL for every child" do
|
328
|
+
Process.expects(:pid).returns(12346).at_least(1)
|
329
|
+
Process.expects(:kill).with(0, 12345).at_least(1)
|
330
|
+
Process.expects(:kill).with(0, 12347).at_least(1)
|
331
|
+
Process.expects(:kill).with("SIGKILL", 12345)
|
332
|
+
Process.expects(:kill).with("SIGKILL", 12347)
|
333
|
+
@worker_class.child_pids << 12345
|
334
|
+
@worker_class.child_pids << 12347
|
335
|
+
assert_equal [12345, 12347], @worker_class.child_pids
|
336
|
+
@worker_class.kill_forks
|
337
|
+
@worker_class.child_pids.clear
|
338
|
+
end
|
339
|
+
|
340
|
+
it "finish_forks should call stop_forks, kill_forks and Process.waitall" do
|
341
|
+
Process.expects(:pid).returns(12346).at_least(1)
|
342
|
+
Process.expects(:kill).with(0, 12345).at_least(1)
|
343
|
+
Process.expects(:kill).with(0, 12347).at_least(1)
|
344
|
+
Process.expects(:kill).with("SIGTERM", 12345)
|
345
|
+
Process.expects(:kill).with("SIGTERM", 12347)
|
346
|
+
Process.expects(:kill).with("SIGKILL", 12345)
|
347
|
+
Process.expects(:kill).with("SIGKILL", 12347)
|
348
|
+
Kernel.expects(:sleep).with(1)
|
349
|
+
Process.expects(:waitall)
|
350
|
+
@worker_class.child_pids << 12345
|
351
|
+
@worker_class.child_pids << 12347
|
352
|
+
assert_equal [12345, 12347], @worker_class.child_pids
|
353
|
+
silenced do
|
354
|
+
@worker_class.finish_forks
|
355
|
+
end
|
356
|
+
@worker_class.child_pids.clear
|
357
|
+
end
|
358
|
+
|
359
|
+
it "finish_forks should not do anything if is_child" do
|
360
|
+
@worker_class.expects(:stop_forks).never
|
361
|
+
@worker_class.is_child = true
|
362
|
+
@worker_class.child_pids << 12345
|
363
|
+
silenced do
|
364
|
+
@worker_class.finish_forks
|
365
|
+
end
|
366
|
+
end
|
367
|
+
|
368
|
+
end # cleanup on parent
|
369
|
+
|
370
|
+
describe "practical tests" do
|
371
|
+
|
372
|
+
before do
|
373
|
+
@templogger = Templogger.new('/tmp')
|
374
|
+
Backburner.configure { |config| config.logger = @templogger.logger }
|
375
|
+
$worker_test_count = 0
|
376
|
+
$worker_success = false
|
377
|
+
$worker_raise = false
|
378
|
+
clear_jobs!('response')
|
379
|
+
clear_jobs!('foo.bar.1', 'foo.bar.2', 'foo.bar.3', 'foo.bar.4', 'foo.bar.5', 'foo.bar.6', 'foo.bar.7')
|
380
|
+
@worker_class.threads_number = 1
|
381
|
+
@worker_class.garbage_after = 10
|
382
|
+
|
383
|
+
silenced do
|
384
|
+
@response_worker = @worker_class.new('response')
|
385
|
+
@response_worker.watch_tube('demo.test.response')
|
386
|
+
end
|
387
|
+
|
388
|
+
@ignore_forks = true
|
389
|
+
end
|
390
|
+
|
391
|
+
after do
|
392
|
+
@templogger.close
|
393
|
+
clear_jobs!('response')
|
394
|
+
clear_jobs!('foo.bar.1', 'foo.bar.2', 'foo.bar.3', 'foo.bar.4', 'foo.bar.5', 'foo.bar.6', 'foo.bar.7')
|
395
|
+
@worker_class.threads_number = 1
|
396
|
+
@worker_class.shutdown = true
|
397
|
+
silenced do
|
398
|
+
@worker_class.stop_forks
|
399
|
+
Timeout::timeout(2) { sleep 0.1 while @worker_class.child_pids.length > 0 }
|
400
|
+
@worker_class.kill_forks
|
401
|
+
Timeout::timeout(2) { sleep 0.1 while @worker_class.child_pids.length > 0 }
|
402
|
+
end
|
403
|
+
end
|
404
|
+
|
405
|
+
it "should work an enqueued job" do
|
406
|
+
@worker = @worker_class.new('foo.bar.1')
|
407
|
+
@worker.start(false)
|
408
|
+
@worker_class.enqueue TestJobFork, [1, 2], :queue => "foo.bar.1"
|
409
|
+
|
410
|
+
silenced do
|
411
|
+
@templogger.wait_for_match(/Completed TestJobFork/m)
|
412
|
+
@response_worker.work_one_job
|
413
|
+
end
|
414
|
+
assert_equal 3, $worker_test_count
|
415
|
+
end # enqueue
|
416
|
+
|
417
|
+
it "should work for an async job" do
|
418
|
+
@worker = @worker_class.new('foo.bar.2')
|
419
|
+
@worker.start(false)
|
420
|
+
TestAsyncJobFork.async(:queue => 'foo.bar.2').foo(3, 5)
|
421
|
+
silenced(2) do
|
422
|
+
@templogger.wait_for_match(/Completed TestAsyncJobFork/m)
|
423
|
+
@response_worker.work_one_job
|
424
|
+
end
|
425
|
+
assert_equal 15, $worker_test_count
|
426
|
+
end # async
|
427
|
+
|
428
|
+
it "should fail quietly if there's an argument error" do
|
429
|
+
@worker = @worker_class.new('foo.bar.3')
|
430
|
+
@worker.start(false)
|
431
|
+
@worker_class.enqueue TestJobFork, ["bam", "foo", "bar"], :queue => "foo.bar.3"
|
432
|
+
silenced(5) do
|
433
|
+
@templogger.wait_for_match(/Finished TestJobFork.*attempt 1 of 1/m)
|
434
|
+
end
|
435
|
+
assert_match(/Exception ArgumentError/, @templogger.body)
|
436
|
+
assert_equal 0, $worker_test_count
|
437
|
+
end # fail, argument
|
438
|
+
|
439
|
+
it "should support retrying jobs and burying" do
|
440
|
+
Backburner.configure { |config| config.max_job_retries = 1; config.retry_delay = 0 }
|
441
|
+
@worker = @worker_class.new('foo.bar.4')
|
442
|
+
@worker.start(false)
|
443
|
+
@worker_class.enqueue TestRetryJobFork, ["bam", "foo"], :queue => 'foo.bar.4'
|
444
|
+
silenced(2) do
|
445
|
+
@templogger.wait_for_match(/Finished TestRetryJobFork.*attempt 2 of 2/m)
|
446
|
+
2.times { @response_worker.work_one_job }
|
447
|
+
end
|
448
|
+
assert_equal 2, $worker_test_count
|
449
|
+
assert_equal false, $worker_success
|
450
|
+
end # retry, bury
|
451
|
+
|
452
|
+
it "should support retrying jobs and succeeds" do
|
453
|
+
Backburner.configure { |config| config.max_job_retries = 2; config.retry_delay = 0 }
|
454
|
+
@worker = @worker_class.new('foo.bar.5')
|
455
|
+
@worker.start(false)
|
456
|
+
@worker_class.enqueue TestRetryJobFork, ["bam", "foo"], :queue => 'foo.bar.5'
|
457
|
+
silenced(2) do
|
458
|
+
@templogger.wait_for_match(/Completed TestRetryJobFork/m)
|
459
|
+
3.times { @response_worker.work_one_job }
|
460
|
+
end
|
461
|
+
assert_equal 3, $worker_test_count
|
462
|
+
assert_equal true, $worker_success
|
463
|
+
end # retrying, succeeds
|
464
|
+
|
465
|
+
it "should support a multithreaded worker without deadlocks" do
|
466
|
+
num_threads = 15
|
467
|
+
num_jobs = 8
|
468
|
+
num_jobs.times do |i|
|
469
|
+
@worker_class.enqueue TestJobMultithreadFork, [6,2], :queue => 'foo.bar.6'
|
470
|
+
end
|
471
|
+
|
472
|
+
@worker_class.threads_number = num_threads
|
473
|
+
@worker = @worker_class.new('foo.bar.6')
|
474
|
+
@worker.start(false)
|
475
|
+
|
476
|
+
silenced do
|
477
|
+
@templogger.wait_for_match(/Completed TestJobMultithreadFork/m)
|
478
|
+
num_jobs.times { @response_worker.work_one_job }
|
479
|
+
end
|
480
|
+
assert_equal num_jobs, $worker_test_count
|
481
|
+
end # multithreaded
|
482
|
+
end # practical tests
|
483
|
+
end # forking and threading
|
484
|
+
end # Backburner::Workers::ThreadsOnFork module
|