backburner 1.1.0 → 1.2.0.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/HOOKS.md +16 -5
- data/README.md +8 -6
- data/examples/demo.rb +2 -2
- data/lib/backburner/configuration.rb +1 -1
- data/lib/backburner/connection.rb +104 -10
- data/lib/backburner/hooks.rb +11 -11
- data/lib/backburner/job.rb +6 -1
- data/lib/backburner/version.rb +1 -1
- data/lib/backburner/worker.rb +42 -59
- data/lib/backburner/workers/forking.rb +5 -1
- data/lib/backburner/workers/simple.rb +1 -1
- data/lib/backburner/workers/threads_on_fork.rb +19 -19
- data/test/async_proxy_test.rb +15 -13
- data/test/connection_test.rb +143 -7
- data/test/fixtures/hooked.rb +6 -0
- data/test/fixtures/test_fork_jobs.rb +13 -1
- data/test/helpers/templogger.rb +1 -1
- data/test/hooks_test.rb +7 -0
- data/test/job_test.rb +33 -0
- data/test/test_helper.rb +30 -8
- data/test/worker_test.rb +41 -41
- data/test/workers/forking_worker_test.rb +10 -13
- data/test/workers/simple_worker_test.rb +23 -10
- data/test/workers/threads_on_fork_worker_test.rb +20 -16
- metadata +16 -17
- data/.DS_Store +0 -0
@@ -29,13 +29,17 @@ module Backburner
|
|
29
29
|
# Waits for a job, works the job, and exits
|
30
30
|
def fork_one_job
|
31
31
|
pid = Process.fork do
|
32
|
-
@connection = Connection.new(Backburner.configuration.beanstalk_url)
|
33
32
|
work_one_job
|
34
33
|
coolest_exit
|
35
34
|
end
|
36
35
|
Process.wait(pid)
|
37
36
|
end
|
38
37
|
|
38
|
+
def on_reconnect(conn)
|
39
|
+
@connection = conn
|
40
|
+
prepare
|
41
|
+
end
|
42
|
+
|
39
43
|
# Exit with Kernel.exit! to avoid at_exit callbacks that should belongs to
|
40
44
|
# parent process
|
41
45
|
# We will use exitcode 99 that means the fork reached the garbage number
|
@@ -50,6 +50,7 @@ module Backburner
|
|
50
50
|
|
51
51
|
def finish_forks
|
52
52
|
return if is_child
|
53
|
+
|
53
54
|
ids = child_pids
|
54
55
|
if ids.length > 0
|
55
56
|
puts "[ThreadsOnFork workers] Stopping forks: #{ids.join(", ")}"
|
@@ -68,7 +69,7 @@ module Backburner
|
|
68
69
|
# Custom initializer just to set @tubes_data
|
69
70
|
def initialize(*args)
|
70
71
|
@tubes_data = {}
|
71
|
-
@
|
72
|
+
@mutex = Mutex.new
|
72
73
|
super
|
73
74
|
self.process_tube_options
|
74
75
|
end
|
@@ -171,8 +172,6 @@ module Backburner
|
|
171
172
|
# If we limit the number of threads to 1 it will just run in a loop without
|
172
173
|
# creating any extra thread.
|
173
174
|
def fork_inner(name)
|
174
|
-
watch_tube(name)
|
175
|
-
|
176
175
|
if @tubes_data[name]
|
177
176
|
queue_config.max_job_retries = @tubes_data[name][:retries] if @tubes_data[name][:retries]
|
178
177
|
else
|
@@ -184,16 +183,18 @@ module Backburner
|
|
184
183
|
@runs = 0
|
185
184
|
|
186
185
|
if @threads_number == 1
|
187
|
-
|
186
|
+
watch_tube(name)
|
187
|
+
run_while_can
|
188
188
|
else
|
189
189
|
threads_count = Thread.list.count
|
190
190
|
@threads_number.times do
|
191
191
|
create_thread do
|
192
|
-
conn = Connection.new(Backburner.configuration.beanstalk_url)
|
193
192
|
begin
|
194
|
-
|
193
|
+
conn = new_connection
|
194
|
+
watch_tube(name, conn)
|
195
|
+
run_while_can(conn)
|
195
196
|
ensure
|
196
|
-
conn.close
|
197
|
+
conn.close if conn
|
197
198
|
end
|
198
199
|
end
|
199
200
|
end
|
@@ -204,21 +205,23 @@ module Backburner
|
|
204
205
|
end
|
205
206
|
|
206
207
|
# Run work_one_job while we can
|
207
|
-
def run_while_can(
|
208
|
-
conn ||= connection
|
209
|
-
watch_tube(name, conn)
|
208
|
+
def run_while_can(conn = connection)
|
210
209
|
while @garbage_after.nil? or @garbage_after > @runs
|
211
|
-
@runs += 1
|
210
|
+
@runs += 1 # FIXME: Likely race condition
|
212
211
|
work_one_job(conn)
|
213
212
|
end
|
214
213
|
end
|
215
214
|
|
216
|
-
# Shortcut for watching a tube on beanstalk connection
|
217
|
-
def watch_tube(name, conn =
|
218
|
-
|
215
|
+
# Shortcut for watching a tube on our beanstalk connection
|
216
|
+
def watch_tube(name, conn = connection)
|
217
|
+
@watching_tube = name
|
219
218
|
conn.tubes.watch!(name)
|
220
219
|
end
|
221
220
|
|
221
|
+
def on_reconnect(conn)
|
222
|
+
watch_tube(@watching_tube, conn) if @watching_tube
|
223
|
+
end
|
224
|
+
|
222
225
|
# Exit with Kernel.exit! to avoid at_exit callbacks that should belongs to
|
223
226
|
# parent process
|
224
227
|
# We will use exitcode 99 that means the fork reached the garbage number
|
@@ -239,20 +242,17 @@ module Backburner
|
|
239
242
|
end
|
240
243
|
|
241
244
|
# Forks the specified block and adds the process to the child process pool
|
245
|
+
# FIXME: If blk.call breaks then the pid isn't added to child_pids and is
|
246
|
+
# never shutdown
|
242
247
|
def fork_it(&blk)
|
243
248
|
pid = Kernel.fork do
|
244
249
|
self.class.is_child = true
|
245
250
|
$0 = "[ThreadsOnFork worker] parent: #{Process.ppid}"
|
246
|
-
@connection = Connection.new(Backburner.configuration.beanstalk_url)
|
247
251
|
blk.call
|
248
252
|
end
|
249
253
|
self.class.child_pids << pid
|
250
254
|
pid
|
251
255
|
end
|
252
|
-
|
253
|
-
def connection
|
254
|
-
@connection || super
|
255
|
-
end
|
256
256
|
end
|
257
257
|
end
|
258
258
|
end
|
data/test/async_proxy_test.rb
CHANGED
@@ -12,23 +12,25 @@ describe "Backburner::AsyncProxy class" do
|
|
12
12
|
should "enqueue job onto worker with no args" do
|
13
13
|
@async = Backburner::AsyncProxy.new(AsyncUser, 10, :pri => 1000, :ttr => 100)
|
14
14
|
@async.foo
|
15
|
-
job, body
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
15
|
+
pop_one_job do |job, body|
|
16
|
+
assert_equal "AsyncUser", body["class"]
|
17
|
+
assert_equal [10, "foo"], body["args"]
|
18
|
+
assert_equal 100, job.ttr
|
19
|
+
assert_equal 1000, job.pri
|
20
|
+
job.delete
|
21
|
+
end
|
21
22
|
end
|
22
23
|
|
23
24
|
should "enqueue job onto worker with args" do
|
24
25
|
@async = Backburner::AsyncProxy.new(AsyncUser, 10, :pri => 1000, :ttr => 100)
|
25
26
|
@async.bar(1, 2, 3)
|
26
|
-
job, body
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
27
|
+
pop_one_job do |job, body|
|
28
|
+
assert_equal "AsyncUser", body["class"]
|
29
|
+
assert_equal [10, "bar", 1, 2, 3], body["args"]
|
30
|
+
assert_equal 100, job.ttr
|
31
|
+
assert_equal 1000, job.pri
|
32
|
+
job.delete
|
33
|
+
end
|
32
34
|
end
|
33
35
|
end # method_missing
|
34
|
-
end # AsyncProxy
|
36
|
+
end # AsyncProxy
|
data/test/connection_test.rb
CHANGED
@@ -3,11 +3,11 @@ require File.expand_path('../test_helper', __FILE__)
|
|
3
3
|
describe "Backburner::Connection class" do
|
4
4
|
describe "for initialize with single url" do
|
5
5
|
before do
|
6
|
-
@connection = Backburner::Connection.new("beanstalk://
|
6
|
+
@connection = Backburner::Connection.new("beanstalk://127.0.0.1")
|
7
7
|
end
|
8
8
|
|
9
9
|
it "should store url in accessor" do
|
10
|
-
assert_equal "beanstalk://
|
10
|
+
assert_equal "beanstalk://127.0.0.1", @connection.url
|
11
11
|
end
|
12
12
|
|
13
13
|
it "should setup beanstalk connection" do
|
@@ -17,9 +17,9 @@ describe "Backburner::Connection class" do
|
|
17
17
|
|
18
18
|
describe "for initialize with url" do
|
19
19
|
it "should delegate the address url correctly" do
|
20
|
-
@connection = Backburner::Connection.new("beanstalk://
|
20
|
+
@connection = Backburner::Connection.new("beanstalk://127.0.0.1")
|
21
21
|
connection = @connection.beanstalk.connection
|
22
|
-
assert_equal '
|
22
|
+
assert_equal '127.0.0.1:11300', connection.address
|
23
23
|
end
|
24
24
|
end # initialize
|
25
25
|
|
@@ -31,13 +31,149 @@ describe "Backburner::Connection class" do
|
|
31
31
|
end
|
32
32
|
end
|
33
33
|
|
34
|
+
describe "for initialize with on_reconnect block" do
|
35
|
+
it "should store the block for use upon reconnect" do
|
36
|
+
callback = proc {}
|
37
|
+
connection = Backburner::Connection.new('beanstalk://127.0.0.1', &callback)
|
38
|
+
assert_equal callback, connection.on_reconnect
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
describe "dealing with connecting and reconnecting" do
|
43
|
+
before do
|
44
|
+
@connection = Backburner::Connection.new('beanstalk://127.0.0.1')
|
45
|
+
end
|
46
|
+
|
47
|
+
it "should know if its connection is open" do
|
48
|
+
assert_equal true, @connection.connected?
|
49
|
+
@connection.close
|
50
|
+
assert_equal false, @connection.connected?
|
51
|
+
end
|
52
|
+
|
53
|
+
it "should be able to attempt reconnecting to beanstalk" do
|
54
|
+
@connection.close
|
55
|
+
assert_equal false, @connection.connected?
|
56
|
+
@connection.reconnect!
|
57
|
+
assert_equal true, @connection.connected?
|
58
|
+
end
|
59
|
+
|
60
|
+
it "should allow for retryable commands" do
|
61
|
+
@result = false
|
62
|
+
@connection.close
|
63
|
+
@connection.retryable { @result = true }
|
64
|
+
assert_equal true, @result
|
65
|
+
end
|
66
|
+
|
67
|
+
it "should provide a hook when a retryable command successfully retries" do
|
68
|
+
@result = false
|
69
|
+
@retried = false
|
70
|
+
@connection.close
|
71
|
+
callback = proc { @result = true }
|
72
|
+
@connection.retryable(:on_retry => callback) do
|
73
|
+
unless @retried
|
74
|
+
@retried = true
|
75
|
+
raise Beaneater::NotConnected.new
|
76
|
+
end
|
77
|
+
end
|
78
|
+
assert_equal true, @result
|
79
|
+
end
|
80
|
+
|
81
|
+
it "should provide a hook when the connection successfully reconnects" do
|
82
|
+
reconnected = false
|
83
|
+
retried = false
|
84
|
+
@connection.close
|
85
|
+
@connection.on_reconnect = proc { reconnected = true }
|
86
|
+
@connection.retryable do
|
87
|
+
unless retried
|
88
|
+
retried = true
|
89
|
+
raise Beaneater::NotConnected.new
|
90
|
+
end
|
91
|
+
end
|
92
|
+
assert_equal true, reconnected
|
93
|
+
end
|
94
|
+
|
95
|
+
it "should call the on_reconnect hook before the on_retry hook" do
|
96
|
+
@result = []
|
97
|
+
@retried = false
|
98
|
+
@connection.close
|
99
|
+
@connection.on_reconnect = proc { @result << "reconnect" }
|
100
|
+
on_retry = proc { @result << "retry" }
|
101
|
+
@connection.retryable(:on_retry => on_retry) do
|
102
|
+
unless @retried
|
103
|
+
@retried = true
|
104
|
+
raise Beaneater::NotConnected.new
|
105
|
+
end
|
106
|
+
end
|
107
|
+
assert_equal %w(reconnect retry), @result
|
108
|
+
end
|
109
|
+
|
110
|
+
describe "ensuring the connection is open" do
|
111
|
+
it "should reattempt the connection to beanstalk several times" do
|
112
|
+
stats = @connection.stats
|
113
|
+
simulate_disconnect(@connection)
|
114
|
+
new_connection = Beaneater.new('127.0.0.1:11300')
|
115
|
+
Beaneater.expects(:new).twice.raises(Beaneater::NotConnected).then.returns(new_connection)
|
116
|
+
@connection.tubes
|
117
|
+
assert_equal true, @connection.connected?
|
118
|
+
end
|
119
|
+
|
120
|
+
it "should not attempt reconnecting if the current connection is open" do
|
121
|
+
assert_equal true, @connection.connected?
|
122
|
+
Beaneater.expects(:new).never
|
123
|
+
@connection.tubes
|
124
|
+
end
|
125
|
+
|
126
|
+
describe "when reconnecting is successful" do
|
127
|
+
it "should allow for a callback" do
|
128
|
+
@result = false
|
129
|
+
simulate_disconnect(@connection)
|
130
|
+
@connection.on_reconnect = proc { @result = true }
|
131
|
+
@connection.tubes
|
132
|
+
assert_equal true, @result
|
133
|
+
end
|
134
|
+
|
135
|
+
it "should pass self to the callback" do
|
136
|
+
result = nil
|
137
|
+
simulate_disconnect(@connection)
|
138
|
+
@connection.on_reconnect = lambda { |conn| result = conn }
|
139
|
+
@connection.tubes
|
140
|
+
assert_equal result, @connection
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
describe "when unable to ensure its connected" do
|
146
|
+
it "should raise Beaneater::NotConnected" do
|
147
|
+
Beaneater.stubs(:new).raises(Beaneater::NotConnected)
|
148
|
+
simulate_disconnect(@connection, 1) # since we're stubbing Beaneater.new above we only to simlulate the disconnect of our current connection
|
149
|
+
assert_raises Beaneater::NotConnected do
|
150
|
+
@connection.tubes
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
describe "when using the retryable method" do
|
156
|
+
it "should yield to the block multiple times" do
|
157
|
+
expected = 2
|
158
|
+
retry_count = 0
|
159
|
+
@connection.retryable(max_retries: expected) do
|
160
|
+
if retry_count < 2
|
161
|
+
retry_count += 1
|
162
|
+
raise Beaneater::NotConnected
|
163
|
+
end
|
164
|
+
end
|
165
|
+
assert_equal expected, retry_count
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
34
170
|
describe "for delegated methods" do
|
35
171
|
before do
|
36
|
-
@connection = Backburner::Connection.new("beanstalk://
|
172
|
+
@connection = Backburner::Connection.new("beanstalk://127.0.0.1")
|
37
173
|
end
|
38
174
|
|
39
175
|
it "delegate methods to beanstalk connection" do
|
40
|
-
assert_equal "
|
176
|
+
assert_equal "127.0.0.1", @connection.connection.host
|
41
177
|
end
|
42
178
|
end # delegator
|
43
|
-
end # Connection
|
179
|
+
end # Connection
|
data/test/fixtures/hooked.rb
CHANGED
@@ -57,4 +57,16 @@ class TestAsyncJobFork
|
|
57
57
|
:worker_test_count_set => x * y
|
58
58
|
}], :queue => 'response'
|
59
59
|
end
|
60
|
-
end
|
60
|
+
end
|
61
|
+
|
62
|
+
class TestJobMultithreadFork
|
63
|
+
include Backburner::Queue
|
64
|
+
queue "test-job-multithread-fork"
|
65
|
+
queue_priority 1000
|
66
|
+
def self.perform(x, y)
|
67
|
+
sleep 1 # simluate work
|
68
|
+
Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
|
69
|
+
:worker_test_count_set => x + y
|
70
|
+
}], :queue => 'response'
|
71
|
+
end
|
72
|
+
end
|
data/test/helpers/templogger.rb
CHANGED
data/test/hooks_test.rb
CHANGED
@@ -83,6 +83,13 @@ describe "Backburner::Hooks module" do
|
|
83
83
|
assert_match(/!!on_bury_foo!! \[10\]/, out)
|
84
84
|
end
|
85
85
|
end
|
86
|
+
|
87
|
+
describe "with on_reconnect" do
|
88
|
+
it "should support successful invocation" do
|
89
|
+
out = silenced { @hooks.invoke_hook_events(HookedWorker.new, :on_reconnect)}
|
90
|
+
assert_match(/!!on_reconnect!!/, out)
|
91
|
+
end
|
92
|
+
end
|
86
93
|
end # invoke_hook_events
|
87
94
|
|
88
95
|
describe "for around_hook_events method" do
|
data/test/job_test.rb
CHANGED
@@ -111,4 +111,37 @@ describe "Backburner::Job module" do
|
|
111
111
|
@job.bury
|
112
112
|
end # bury
|
113
113
|
end # simple delegation
|
114
|
+
|
115
|
+
describe "timing out for various values of ttr" do
|
116
|
+
before do
|
117
|
+
@task_body = { "class" => "NestedDemo::TestJobC", "args" => [56] }
|
118
|
+
end
|
119
|
+
|
120
|
+
describe "when ttr == 0" do
|
121
|
+
it "should use 0 for the timeout" do
|
122
|
+
@task = stub(:body => @task_body, :delete => true, :ttr => 0)
|
123
|
+
@job = Backburner::Job.new(@task)
|
124
|
+
Timeout.expects(:timeout).with(0)
|
125
|
+
@job.process
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
describe "when ttr == 1" do
|
130
|
+
it "should use 1 for the timeout" do
|
131
|
+
@task = stub(:body => @task_body, :delete => true, :ttr => 1)
|
132
|
+
@job = Backburner::Job.new(@task)
|
133
|
+
Timeout.expects(:timeout).with(1)
|
134
|
+
@job.process
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
describe "when ttr > 1" do
|
139
|
+
it "should use ttr-1 for the timeout" do
|
140
|
+
@task = stub(:body => @task_body, :delete => true, :ttr => 2)
|
141
|
+
@job = Backburner::Job.new(@task)
|
142
|
+
Timeout.expects(:timeout).with(1)
|
143
|
+
@job.process
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
114
147
|
end
|
data/test/test_helper.rb
CHANGED
@@ -12,7 +12,7 @@ require File.expand_path('../helpers/templogger', __FILE__)
|
|
12
12
|
|
13
13
|
# Configure Backburner
|
14
14
|
Backburner.configure do |config|
|
15
|
-
config.beanstalk_url = "beanstalk://
|
15
|
+
config.beanstalk_url = "beanstalk://127.0.0.1"
|
16
16
|
config.tube_namespace = "demo.test"
|
17
17
|
end
|
18
18
|
|
@@ -25,7 +25,7 @@ module Kernel
|
|
25
25
|
def capture_stdout
|
26
26
|
if ENV['DEBUG'] # Skip if debug mode
|
27
27
|
yield
|
28
|
-
""
|
28
|
+
return ""
|
29
29
|
end
|
30
30
|
|
31
31
|
out = StringIO.new
|
@@ -87,20 +87,42 @@ class MiniTest::Spec
|
|
87
87
|
Timeout::timeout(time) { capture_stdout(&block) }
|
88
88
|
end
|
89
89
|
|
90
|
+
def beanstalk_connection
|
91
|
+
Backburner::Connection.new(Backburner.configuration.beanstalk_url)
|
92
|
+
end
|
93
|
+
|
90
94
|
# pop_one_job(tube_name)
|
91
|
-
def pop_one_job(tube_name=Backburner.configuration.primary_queue)
|
92
|
-
|
93
|
-
|
95
|
+
def pop_one_job(tube_name=Backburner.configuration.primary_queue, &block)
|
96
|
+
tube_name = [Backburner.configuration.tube_namespace, tube_name].join(".")
|
97
|
+
connection = beanstalk_connection
|
94
98
|
connection.tubes.watch!(tube_name)
|
95
99
|
silenced(3) { @res = connection.tubes.reserve }
|
96
|
-
|
100
|
+
yield @res, JSON.parse(@res.body)
|
101
|
+
ensure
|
102
|
+
connection.close if connection
|
97
103
|
end
|
98
104
|
|
99
105
|
# clear_jobs!('foo')
|
100
106
|
def clear_jobs!(*tube_names)
|
107
|
+
connection = beanstalk_connection
|
101
108
|
tube_names.each do |tube_name|
|
102
109
|
expanded_name = [Backburner.configuration.tube_namespace, tube_name].join(".")
|
103
|
-
|
110
|
+
connection.tubes.find(expanded_name).clear
|
111
|
+
end
|
112
|
+
ensure
|
113
|
+
connection.close if connection
|
114
|
+
end
|
115
|
+
|
116
|
+
# Simulates a broken connection for any Beaneater::Connection. Will
|
117
|
+
# simulate a restored connection after `reconnects_after`. This is expected
|
118
|
+
# to be used when ensuring a Beaneater connection is open, therefore
|
119
|
+
def simulate_disconnect(connection, reconnects_after = 2)
|
120
|
+
connection.beanstalk.connection.connection.expects(:closed? => true)
|
121
|
+
returns = Array.new(reconnects_after - 1, stub('TCPSocket'))
|
122
|
+
returns.each do |socket|
|
123
|
+
result = (socket != returns.last)
|
124
|
+
socket.stubs(:closed? => result)
|
104
125
|
end
|
126
|
+
TCPSocket.expects(:new).times(returns.size).returns(*returns)
|
105
127
|
end
|
106
|
-
end # MiniTest::Spec
|
128
|
+
end # MiniTest::Spec
|