backburner 1.1.0 → 1.2.0.pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -29,13 +29,17 @@ module Backburner
29
29
  # Waits for a job, works the job, and exits
30
30
  def fork_one_job
31
31
  pid = Process.fork do
32
- @connection = Connection.new(Backburner.configuration.beanstalk_url)
33
32
  work_one_job
34
33
  coolest_exit
35
34
  end
36
35
  Process.wait(pid)
37
36
  end
38
37
 
38
+ def on_reconnect(conn)
39
+ @connection = conn
40
+ prepare
41
+ end
42
+
39
43
  # Exit with Kernel.exit! to avoid at_exit callbacks that should belongs to
40
44
  # parent process
41
45
  # We will use exitcode 99 that means the fork reached the garbage number
@@ -26,4 +26,4 @@ module Backburner
26
26
  end
27
27
  end # Basic
28
28
  end # Workers
29
- end # Backburner
29
+ end # Backburner
@@ -50,6 +50,7 @@ module Backburner
50
50
 
51
51
  def finish_forks
52
52
  return if is_child
53
+
53
54
  ids = child_pids
54
55
  if ids.length > 0
55
56
  puts "[ThreadsOnFork workers] Stopping forks: #{ids.join(", ")}"
@@ -68,7 +69,7 @@ module Backburner
68
69
  # Custom initializer just to set @tubes_data
69
70
  def initialize(*args)
70
71
  @tubes_data = {}
71
- @connection = nil
72
+ @mutex = Mutex.new
72
73
  super
73
74
  self.process_tube_options
74
75
  end
@@ -171,8 +172,6 @@ module Backburner
171
172
  # If we limit the number of threads to 1 it will just run in a loop without
172
173
  # creating any extra thread.
173
174
  def fork_inner(name)
174
- watch_tube(name)
175
-
176
175
  if @tubes_data[name]
177
176
  queue_config.max_job_retries = @tubes_data[name][:retries] if @tubes_data[name][:retries]
178
177
  else
@@ -184,16 +183,18 @@ module Backburner
184
183
  @runs = 0
185
184
 
186
185
  if @threads_number == 1
187
- run_while_can(name)
186
+ watch_tube(name)
187
+ run_while_can
188
188
  else
189
189
  threads_count = Thread.list.count
190
190
  @threads_number.times do
191
191
  create_thread do
192
- conn = Connection.new(Backburner.configuration.beanstalk_url)
193
192
  begin
194
- run_while_can(name, conn)
193
+ conn = new_connection
194
+ watch_tube(name, conn)
195
+ run_while_can(conn)
195
196
  ensure
196
- conn.close
197
+ conn.close if conn
197
198
  end
198
199
  end
199
200
  end
@@ -204,21 +205,23 @@ module Backburner
204
205
  end
205
206
 
206
207
  # Run work_one_job while we can
207
- def run_while_can(name, conn = nil)
208
- conn ||= connection
209
- watch_tube(name, conn)
208
+ def run_while_can(conn = connection)
210
209
  while @garbage_after.nil? or @garbage_after > @runs
211
- @runs += 1
210
+ @runs += 1 # FIXME: Likely race condition
212
211
  work_one_job(conn)
213
212
  end
214
213
  end
215
214
 
216
- # Shortcut for watching a tube on beanstalk connection
217
- def watch_tube(name, conn = nil)
218
- conn ||= connection
215
+ # Shortcut for watching a tube on our beanstalk connection
216
+ def watch_tube(name, conn = connection)
217
+ @watching_tube = name
219
218
  conn.tubes.watch!(name)
220
219
  end
221
220
 
221
+ def on_reconnect(conn)
222
+ watch_tube(@watching_tube, conn) if @watching_tube
223
+ end
224
+
222
225
  # Exit with Kernel.exit! to avoid at_exit callbacks that should belongs to
223
226
  # parent process
224
227
  # We will use exitcode 99 that means the fork reached the garbage number
@@ -239,20 +242,17 @@ module Backburner
239
242
  end
240
243
 
241
244
  # Forks the specified block and adds the process to the child process pool
245
+ # FIXME: If blk.call breaks then the pid isn't added to child_pids and is
246
+ # never shutdown
242
247
  def fork_it(&blk)
243
248
  pid = Kernel.fork do
244
249
  self.class.is_child = true
245
250
  $0 = "[ThreadsOnFork worker] parent: #{Process.ppid}"
246
- @connection = Connection.new(Backburner.configuration.beanstalk_url)
247
251
  blk.call
248
252
  end
249
253
  self.class.child_pids << pid
250
254
  pid
251
255
  end
252
-
253
- def connection
254
- @connection || super
255
- end
256
256
  end
257
257
  end
258
258
  end
@@ -12,23 +12,25 @@ describe "Backburner::AsyncProxy class" do
12
12
  should "enqueue job onto worker with no args" do
13
13
  @async = Backburner::AsyncProxy.new(AsyncUser, 10, :pri => 1000, :ttr => 100)
14
14
  @async.foo
15
- job, body = pop_one_job
16
- assert_equal "AsyncUser", body["class"]
17
- assert_equal [10, "foo"], body["args"]
18
- assert_equal 100, job.ttr
19
- assert_equal 1000, job.pri
20
- job.delete
15
+ pop_one_job do |job, body|
16
+ assert_equal "AsyncUser", body["class"]
17
+ assert_equal [10, "foo"], body["args"]
18
+ assert_equal 100, job.ttr
19
+ assert_equal 1000, job.pri
20
+ job.delete
21
+ end
21
22
  end
22
23
 
23
24
  should "enqueue job onto worker with args" do
24
25
  @async = Backburner::AsyncProxy.new(AsyncUser, 10, :pri => 1000, :ttr => 100)
25
26
  @async.bar(1, 2, 3)
26
- job, body = pop_one_job
27
- assert_equal "AsyncUser", body["class"]
28
- assert_equal [10, "bar", 1, 2, 3], body["args"]
29
- assert_equal 100, job.ttr
30
- assert_equal 1000, job.pri
31
- job.delete
27
+ pop_one_job do |job, body|
28
+ assert_equal "AsyncUser", body["class"]
29
+ assert_equal [10, "bar", 1, 2, 3], body["args"]
30
+ assert_equal 100, job.ttr
31
+ assert_equal 1000, job.pri
32
+ job.delete
33
+ end
32
34
  end
33
35
  end # method_missing
34
- end # AsyncProxy
36
+ end # AsyncProxy
@@ -3,11 +3,11 @@ require File.expand_path('../test_helper', __FILE__)
3
3
  describe "Backburner::Connection class" do
4
4
  describe "for initialize with single url" do
5
5
  before do
6
- @connection = Backburner::Connection.new("beanstalk://localhost")
6
+ @connection = Backburner::Connection.new("beanstalk://127.0.0.1")
7
7
  end
8
8
 
9
9
  it "should store url in accessor" do
10
- assert_equal "beanstalk://localhost", @connection.url
10
+ assert_equal "beanstalk://127.0.0.1", @connection.url
11
11
  end
12
12
 
13
13
  it "should setup beanstalk connection" do
@@ -17,9 +17,9 @@ describe "Backburner::Connection class" do
17
17
 
18
18
  describe "for initialize with url" do
19
19
  it "should delegate the address url correctly" do
20
- @connection = Backburner::Connection.new("beanstalk://localhost")
20
+ @connection = Backburner::Connection.new("beanstalk://127.0.0.1")
21
21
  connection = @connection.beanstalk.connection
22
- assert_equal 'localhost:11300', connection.address
22
+ assert_equal '127.0.0.1:11300', connection.address
23
23
  end
24
24
  end # initialize
25
25
 
@@ -31,13 +31,149 @@ describe "Backburner::Connection class" do
31
31
  end
32
32
  end
33
33
 
34
+ describe "for initialize with on_reconnect block" do
35
+ it "should store the block for use upon reconnect" do
36
+ callback = proc {}
37
+ connection = Backburner::Connection.new('beanstalk://127.0.0.1', &callback)
38
+ assert_equal callback, connection.on_reconnect
39
+ end
40
+ end
41
+
42
+ describe "dealing with connecting and reconnecting" do
43
+ before do
44
+ @connection = Backburner::Connection.new('beanstalk://127.0.0.1')
45
+ end
46
+
47
+ it "should know if its connection is open" do
48
+ assert_equal true, @connection.connected?
49
+ @connection.close
50
+ assert_equal false, @connection.connected?
51
+ end
52
+
53
+ it "should be able to attempt reconnecting to beanstalk" do
54
+ @connection.close
55
+ assert_equal false, @connection.connected?
56
+ @connection.reconnect!
57
+ assert_equal true, @connection.connected?
58
+ end
59
+
60
+ it "should allow for retryable commands" do
61
+ @result = false
62
+ @connection.close
63
+ @connection.retryable { @result = true }
64
+ assert_equal true, @result
65
+ end
66
+
67
+ it "should provide a hook when a retryable command successfully retries" do
68
+ @result = false
69
+ @retried = false
70
+ @connection.close
71
+ callback = proc { @result = true }
72
+ @connection.retryable(:on_retry => callback) do
73
+ unless @retried
74
+ @retried = true
75
+ raise Beaneater::NotConnected.new
76
+ end
77
+ end
78
+ assert_equal true, @result
79
+ end
80
+
81
+ it "should provide a hook when the connection successfully reconnects" do
82
+ reconnected = false
83
+ retried = false
84
+ @connection.close
85
+ @connection.on_reconnect = proc { reconnected = true }
86
+ @connection.retryable do
87
+ unless retried
88
+ retried = true
89
+ raise Beaneater::NotConnected.new
90
+ end
91
+ end
92
+ assert_equal true, reconnected
93
+ end
94
+
95
+ it "should call the on_reconnect hook before the on_retry hook" do
96
+ @result = []
97
+ @retried = false
98
+ @connection.close
99
+ @connection.on_reconnect = proc { @result << "reconnect" }
100
+ on_retry = proc { @result << "retry" }
101
+ @connection.retryable(:on_retry => on_retry) do
102
+ unless @retried
103
+ @retried = true
104
+ raise Beaneater::NotConnected.new
105
+ end
106
+ end
107
+ assert_equal %w(reconnect retry), @result
108
+ end
109
+
110
+ describe "ensuring the connection is open" do
111
+ it "should reattempt the connection to beanstalk several times" do
112
+ stats = @connection.stats
113
+ simulate_disconnect(@connection)
114
+ new_connection = Beaneater.new('127.0.0.1:11300')
115
+ Beaneater.expects(:new).twice.raises(Beaneater::NotConnected).then.returns(new_connection)
116
+ @connection.tubes
117
+ assert_equal true, @connection.connected?
118
+ end
119
+
120
+ it "should not attempt reconnecting if the current connection is open" do
121
+ assert_equal true, @connection.connected?
122
+ Beaneater.expects(:new).never
123
+ @connection.tubes
124
+ end
125
+
126
+ describe "when reconnecting is successful" do
127
+ it "should allow for a callback" do
128
+ @result = false
129
+ simulate_disconnect(@connection)
130
+ @connection.on_reconnect = proc { @result = true }
131
+ @connection.tubes
132
+ assert_equal true, @result
133
+ end
134
+
135
+ it "should pass self to the callback" do
136
+ result = nil
137
+ simulate_disconnect(@connection)
138
+ @connection.on_reconnect = lambda { |conn| result = conn }
139
+ @connection.tubes
140
+ assert_equal result, @connection
141
+ end
142
+ end
143
+ end
144
+
145
+ describe "when unable to ensure its connected" do
146
+ it "should raise Beaneater::NotConnected" do
147
+ Beaneater.stubs(:new).raises(Beaneater::NotConnected)
148
+ simulate_disconnect(@connection, 1) # since we're stubbing Beaneater.new above we only to simlulate the disconnect of our current connection
149
+ assert_raises Beaneater::NotConnected do
150
+ @connection.tubes
151
+ end
152
+ end
153
+ end
154
+
155
+ describe "when using the retryable method" do
156
+ it "should yield to the block multiple times" do
157
+ expected = 2
158
+ retry_count = 0
159
+ @connection.retryable(max_retries: expected) do
160
+ if retry_count < 2
161
+ retry_count += 1
162
+ raise Beaneater::NotConnected
163
+ end
164
+ end
165
+ assert_equal expected, retry_count
166
+ end
167
+ end
168
+ end
169
+
34
170
  describe "for delegated methods" do
35
171
  before do
36
- @connection = Backburner::Connection.new("beanstalk://localhost")
172
+ @connection = Backburner::Connection.new("beanstalk://127.0.0.1")
37
173
  end
38
174
 
39
175
  it "delegate methods to beanstalk connection" do
40
- assert_equal "localhost", @connection.connection.host
176
+ assert_equal "127.0.0.1", @connection.connection.host
41
177
  end
42
178
  end # delegator
43
- end # Connection
179
+ end # Connection
@@ -114,3 +114,9 @@ class HookedObjectSuccess
114
114
  puts "This is the job running successfully!! #{x.inspect}"
115
115
  end
116
116
  end # HookedObjectSuccess
117
+
118
+ class HookedWorker < Backburner::Worker
119
+ def on_reconnect
120
+ puts "!!on_reconnect!!"
121
+ end
122
+ end
@@ -57,4 +57,16 @@ class TestAsyncJobFork
57
57
  :worker_test_count_set => x * y
58
58
  }], :queue => 'response'
59
59
  end
60
- end
60
+ end
61
+
62
+ class TestJobMultithreadFork
63
+ include Backburner::Queue
64
+ queue "test-job-multithread-fork"
65
+ queue_priority 1000
66
+ def self.perform(x, y)
67
+ sleep 1 # simluate work
68
+ Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
69
+ :worker_test_count_set => x + y
70
+ }], :queue => 'response'
71
+ end
72
+ end
@@ -19,4 +19,4 @@ class Templogger
19
19
  def close
20
20
  @file.close
21
21
  end
22
- end
22
+ end
@@ -83,6 +83,13 @@ describe "Backburner::Hooks module" do
83
83
  assert_match(/!!on_bury_foo!! \[10\]/, out)
84
84
  end
85
85
  end
86
+
87
+ describe "with on_reconnect" do
88
+ it "should support successful invocation" do
89
+ out = silenced { @hooks.invoke_hook_events(HookedWorker.new, :on_reconnect)}
90
+ assert_match(/!!on_reconnect!!/, out)
91
+ end
92
+ end
86
93
  end # invoke_hook_events
87
94
 
88
95
  describe "for around_hook_events method" do
@@ -111,4 +111,37 @@ describe "Backburner::Job module" do
111
111
  @job.bury
112
112
  end # bury
113
113
  end # simple delegation
114
+
115
+ describe "timing out for various values of ttr" do
116
+ before do
117
+ @task_body = { "class" => "NestedDemo::TestJobC", "args" => [56] }
118
+ end
119
+
120
+ describe "when ttr == 0" do
121
+ it "should use 0 for the timeout" do
122
+ @task = stub(:body => @task_body, :delete => true, :ttr => 0)
123
+ @job = Backburner::Job.new(@task)
124
+ Timeout.expects(:timeout).with(0)
125
+ @job.process
126
+ end
127
+ end
128
+
129
+ describe "when ttr == 1" do
130
+ it "should use 1 for the timeout" do
131
+ @task = stub(:body => @task_body, :delete => true, :ttr => 1)
132
+ @job = Backburner::Job.new(@task)
133
+ Timeout.expects(:timeout).with(1)
134
+ @job.process
135
+ end
136
+ end
137
+
138
+ describe "when ttr > 1" do
139
+ it "should use ttr-1 for the timeout" do
140
+ @task = stub(:body => @task_body, :delete => true, :ttr => 2)
141
+ @job = Backburner::Job.new(@task)
142
+ Timeout.expects(:timeout).with(1)
143
+ @job.process
144
+ end
145
+ end
146
+ end
114
147
  end
@@ -12,7 +12,7 @@ require File.expand_path('../helpers/templogger', __FILE__)
12
12
 
13
13
  # Configure Backburner
14
14
  Backburner.configure do |config|
15
- config.beanstalk_url = "beanstalk://localhost"
15
+ config.beanstalk_url = "beanstalk://127.0.0.1"
16
16
  config.tube_namespace = "demo.test"
17
17
  end
18
18
 
@@ -25,7 +25,7 @@ module Kernel
25
25
  def capture_stdout
26
26
  if ENV['DEBUG'] # Skip if debug mode
27
27
  yield
28
- ""
28
+ return ""
29
29
  end
30
30
 
31
31
  out = StringIO.new
@@ -87,20 +87,42 @@ class MiniTest::Spec
87
87
  Timeout::timeout(time) { capture_stdout(&block) }
88
88
  end
89
89
 
90
+ def beanstalk_connection
91
+ Backburner::Connection.new(Backburner.configuration.beanstalk_url)
92
+ end
93
+
90
94
  # pop_one_job(tube_name)
91
- def pop_one_job(tube_name=Backburner.configuration.primary_queue)
92
- connection = Backburner::Worker.connection
93
- tube_name = [Backburner.configuration.tube_namespace, tube_name].join(".")
95
+ def pop_one_job(tube_name=Backburner.configuration.primary_queue, &block)
96
+ tube_name = [Backburner.configuration.tube_namespace, tube_name].join(".")
97
+ connection = beanstalk_connection
94
98
  connection.tubes.watch!(tube_name)
95
99
  silenced(3) { @res = connection.tubes.reserve }
96
- return @res, JSON.parse(@res.body)
100
+ yield @res, JSON.parse(@res.body)
101
+ ensure
102
+ connection.close if connection
97
103
  end
98
104
 
99
105
  # clear_jobs!('foo')
100
106
  def clear_jobs!(*tube_names)
107
+ connection = beanstalk_connection
101
108
  tube_names.each do |tube_name|
102
109
  expanded_name = [Backburner.configuration.tube_namespace, tube_name].join(".")
103
- Backburner::Worker.connection.tubes.find(expanded_name).clear
110
+ connection.tubes.find(expanded_name).clear
111
+ end
112
+ ensure
113
+ connection.close if connection
114
+ end
115
+
116
+ # Simulates a broken connection for any Beaneater::Connection. Will
117
+ # simulate a restored connection after `reconnects_after`. This is expected
118
+ # to be used when ensuring a Beaneater connection is open, therefore
119
+ def simulate_disconnect(connection, reconnects_after = 2)
120
+ connection.beanstalk.connection.connection.expects(:closed? => true)
121
+ returns = Array.new(reconnects_after - 1, stub('TCPSocket'))
122
+ returns.each do |socket|
123
+ result = (socket != returns.last)
124
+ socket.stubs(:closed? => result)
104
125
  end
126
+ TCPSocket.expects(:new).times(returns.size).returns(*returns)
105
127
  end
106
- end # MiniTest::Spec
128
+ end # MiniTest::Spec