beaneater 1.1.3 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +44 -0
- data/CHANGELOG.md +23 -0
- data/Gemfile +2 -0
- data/README.md +29 -1
- data/Rakefile +2 -0
- data/beaneater.gemspec +2 -0
- data/examples/demo.rb +2 -0
- data/lib/beaneater/configuration.rb +10 -0
- data/lib/beaneater/connection.rb +315 -57
- data/lib/beaneater/errors.rb +16 -0
- data/lib/beaneater/job/collection.rb +44 -6
- data/lib/beaneater/job/record.rb +2 -0
- data/lib/beaneater/job.rb +2 -0
- data/lib/beaneater/stats/fast_struct.rb +2 -0
- data/lib/beaneater/stats/stat_struct.rb +8 -1
- data/lib/beaneater/stats.rb +2 -0
- data/lib/beaneater/tube/collection.rb +2 -0
- data/lib/beaneater/tube/record.rb +2 -0
- data/lib/beaneater/tube.rb +2 -0
- data/lib/beaneater/version.rb +3 -1
- data/lib/beaneater.rb +2 -0
- data/test/beaneater_test.rb +2 -0
- data/test/connection_test.rb +612 -12
- data/test/errors_test.rb +2 -0
- data/test/job_test.rb +14 -0
- data/test/jobs_test.rb +431 -0
- data/test/prompt_regexp_test.rb +2 -0
- data/test/stat_struct_test.rb +7 -0
- data/test/stats_test.rb +2 -0
- data/test/test_helper.rb +47 -0
- data/test/tube_test.rb +125 -0
- data/test/tubes_test.rb +18 -1
- metadata +5 -4
data/test/job_test.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
# test/job_test.rb
|
|
2
4
|
|
|
3
5
|
require File.expand_path('../test_helper', __FILE__)
|
|
@@ -10,6 +12,7 @@ describe Beaneater::Job do
|
|
|
10
12
|
|
|
11
13
|
describe "for #bury" do
|
|
12
14
|
before do
|
|
15
|
+
@tube.clear
|
|
13
16
|
@time = Time.now.to_i
|
|
14
17
|
@tube.put "foo bury #{@time}", :pri => 5
|
|
15
18
|
end
|
|
@@ -205,6 +208,17 @@ describe Beaneater::Job do
|
|
|
205
208
|
assert_equal @tube.name, job.tube
|
|
206
209
|
job.release
|
|
207
210
|
end
|
|
211
|
+
|
|
212
|
+
it("should keep dashes and dots in the tube name") do
|
|
213
|
+
tube = @beanstalk.tubes.find('email-jobs.v1')
|
|
214
|
+
tube.put 'bar'
|
|
215
|
+
job = tube.reserve
|
|
216
|
+
assert_equal 'email-jobs.v1', job.tube
|
|
217
|
+
assert_equal 'email-jobs.v1', job.stats.tube
|
|
218
|
+
assert_equal 'reserved', job.stats.state
|
|
219
|
+
assert_kind_of Integer, job.stats.id
|
|
220
|
+
job.delete
|
|
221
|
+
end
|
|
208
222
|
end # tube
|
|
209
223
|
|
|
210
224
|
describe "for #pri" do
|
data/test/jobs_test.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
# test/jobs_test.rb
|
|
2
4
|
|
|
3
5
|
require File.expand_path('../test_helper', __FILE__)
|
|
@@ -104,4 +106,433 @@ describe Beaneater::Jobs do
|
|
|
104
106
|
assert_equal 1, @beanstalk.tubes.find('tube_buried').stats.current_jobs_buried
|
|
105
107
|
end
|
|
106
108
|
end # for_process!
|
|
109
|
+
|
|
110
|
+
describe "for process! with a dashed tube name" do
|
|
111
|
+
before do
|
|
112
|
+
$ran = []
|
|
113
|
+
cleanup_tubes!(['email-jobs', 'tube_stop'])
|
|
114
|
+
@jobs.register('email-jobs') { |job| $ran << job.body }
|
|
115
|
+
@jobs.register('tube_stop') { |job| $ran << job.body; @jobs.stop! }
|
|
116
|
+
@beanstalk.tubes.find('email-jobs').put('email', :pri => 1)
|
|
117
|
+
@beanstalk.tubes.find('tube_stop').put('stop', :pri => 2)
|
|
118
|
+
@jobs.process!(:release_delay => 0)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
it "should run the handler registered for the tube" do
|
|
122
|
+
assert_equal ['email', 'stop'], $ran
|
|
123
|
+
assert_nil @beanstalk.tubes.find('email-jobs').peek(:buried)
|
|
124
|
+
end
|
|
125
|
+
end # process! with a dashed tube name
|
|
126
|
+
|
|
127
|
+
describe "for process! when a failing handler no longer holds its job" do
|
|
128
|
+
before do
|
|
129
|
+
$foo = []
|
|
130
|
+
cleanup_tubes!(['tube_lost'])
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
after do
|
|
134
|
+
cleanup_tubes!(['tube_lost'])
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# The handler loses 'first', puts 'second' and raises. 'second' outranks 'first', so it is
|
|
138
|
+
# reserved next whatever happened to 'first'.
|
|
139
|
+
def process_until_second_job(ttr = 120, &lose_job)
|
|
140
|
+
tube = @beanstalk.tubes.find('tube_lost')
|
|
141
|
+
@jobs.register('tube_lost') do |job|
|
|
142
|
+
$foo << job.body
|
|
143
|
+
if job.body == 'second'
|
|
144
|
+
@jobs.stop!
|
|
145
|
+
else
|
|
146
|
+
lose_job.call(job)
|
|
147
|
+
tube.put('second', :pri => 0)
|
|
148
|
+
raise RuntimeError
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
tube.put('first', :pri => 1, :ttr => ttr)
|
|
152
|
+
Timeout.timeout(15) { @jobs.process!(:reserve_timeout => 1) }
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
it "keeps processing when the handler deleted the job" do
|
|
156
|
+
process_until_second_job { |job| job.delete }
|
|
157
|
+
assert_equal ['first', 'second'], $foo
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
it "keeps processing when the handler released the job" do
|
|
161
|
+
process_until_second_job { |job| job.release(:delay => 5) }
|
|
162
|
+
assert_equal ['first', 'second'], $foo
|
|
163
|
+
assert_equal 1, @beanstalk.tubes.find('tube_lost').stats.current_jobs_delayed
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
it "keeps processing when the handler overran the job's TTR" do
|
|
167
|
+
process_until_second_job(1) do |job|
|
|
168
|
+
# the server releases a job some time after its TTR runs out
|
|
169
|
+
deadline = Time.now + 10
|
|
170
|
+
sleep 0.1 while job.stats.state == 'reserved' && Time.now < deadline
|
|
171
|
+
refute_equal 'reserved', job.stats.state
|
|
172
|
+
end
|
|
173
|
+
assert_equal ['first', 'second'], $foo
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
describe "for process! with stop!" do
|
|
178
|
+
before do
|
|
179
|
+
cleanup_tubes!(['tube_stop'])
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
after do
|
|
183
|
+
cleanup_tubes!(['tube_stop'])
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# Runs process! in a thread and yields. Returns :returned, the exception process! raised, or
|
|
187
|
+
# :still_running if it hadn't finished within seconds.
|
|
188
|
+
def process_in_thread(seconds)
|
|
189
|
+
worker = Thread.new do
|
|
190
|
+
begin
|
|
191
|
+
@jobs.process!(:reserve_timeout => 1)
|
|
192
|
+
:returned
|
|
193
|
+
rescue StandardError => ex
|
|
194
|
+
ex
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
yield if block_given?
|
|
198
|
+
return worker.value if worker.join(seconds)
|
|
199
|
+
worker.kill
|
|
200
|
+
:still_running
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
it "returns within one reserve_timeout when stop! is called while idle" do
|
|
204
|
+
@jobs.register('tube_stop') { |job| }
|
|
205
|
+
result = process_in_thread(3) do
|
|
206
|
+
sleep 0.5
|
|
207
|
+
@jobs.stop!
|
|
208
|
+
end
|
|
209
|
+
assert_equal :returned, result
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
it "returns when stop! is called by a handler whose job another worker took over" do
|
|
213
|
+
other = Beaneater.new('localhost')
|
|
214
|
+
other.tubes.watch!('tube_stop')
|
|
215
|
+
runs = 0
|
|
216
|
+
@jobs.register('tube_stop') do |job|
|
|
217
|
+
runs += 1
|
|
218
|
+
job.release # as the server does once a job's TTR runs out
|
|
219
|
+
other.tubes.reserve(1)
|
|
220
|
+
@jobs.stop!
|
|
221
|
+
end
|
|
222
|
+
@beanstalk.tubes.find('tube_stop').put('taken over')
|
|
223
|
+
|
|
224
|
+
assert_equal :returned, process_in_thread(5)
|
|
225
|
+
assert_equal 1, runs
|
|
226
|
+
ensure
|
|
227
|
+
other.close if other
|
|
228
|
+
end
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
describe "for process! when the job_parser rejects a job" do
|
|
232
|
+
before do
|
|
233
|
+
cleanup_tubes!(['tube_bad_body'])
|
|
234
|
+
Beaneater.configure.job_parser = lambda { |body| JSON.parse(body) }
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
after do
|
|
238
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
239
|
+
cleanup_tubes!(['tube_bad_body'])
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
it "buries the job and goes on to the next one" do
|
|
243
|
+
handled = []
|
|
244
|
+
@jobs.register('tube_bad_body') do |job|
|
|
245
|
+
handled << job.body
|
|
246
|
+
@jobs.stop!
|
|
247
|
+
end
|
|
248
|
+
tube = @beanstalk.tubes.find('tube_bad_body')
|
|
249
|
+
tube.put('not json', :pri => 1)
|
|
250
|
+
tube.put('{"ok":1}', :pri => 2)
|
|
251
|
+
|
|
252
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1) }
|
|
253
|
+
assert_equal [{ 'ok' => 1 }], handled
|
|
254
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
255
|
+
buried = tube.peek(:buried)
|
|
256
|
+
refute_nil buried, "the job the parser rejected was not buried"
|
|
257
|
+
assert_equal 'not json', buried.body
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
# process! goes on after NotFoundError (and JobNotReserved, TimedOutError); the job it had
|
|
261
|
+
# must not be taken for the bad job that follows.
|
|
262
|
+
def put_after_retry_jobs
|
|
263
|
+
tube = @beanstalk.tubes.find('tube_bad_body')
|
|
264
|
+
first = tube.put('{"first":1}', :pri => 1)[:id]
|
|
265
|
+
bad = tube.put('not json', :pri => 2)[:id]
|
|
266
|
+
tube.put('{"stop":1}', :pri => 3)
|
|
267
|
+
[tube, first, bad]
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
def stats_of(id)
|
|
271
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
272
|
+
@beanstalk.connection.transmit("stats-job #{id}")[:body]
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
it "buries the bad job after a handler deleted its own job" do
|
|
276
|
+
@jobs.register('tube_bad_body') do |job|
|
|
277
|
+
job.body == { 'stop' => 1 } ? @jobs.stop! : job.delete # process!'s own delete then gets NOT_FOUND
|
|
278
|
+
end
|
|
279
|
+
tube, _first, bad = put_after_retry_jobs
|
|
280
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1) }
|
|
281
|
+
assert_equal 'buried', stats_of(bad)['state']
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
it "buries both the failed job and the bad job after a handler raised NotFoundError" do
|
|
285
|
+
@jobs.register('tube_bad_body') do |job|
|
|
286
|
+
if job.body == { 'stop' => 1 }
|
|
287
|
+
@jobs.stop!
|
|
288
|
+
else
|
|
289
|
+
@beanstalk.connection.transmit('delete 999999') # NOT_FOUND for some other job
|
|
290
|
+
end
|
|
291
|
+
end
|
|
292
|
+
tube, first, bad = put_after_retry_jobs
|
|
293
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1) }
|
|
294
|
+
assert_equal 'buried', stats_of(bad)['state']
|
|
295
|
+
assert_equal 'buried', stats_of(first)['state'] # its handler failed, with the job still reserved
|
|
296
|
+
ensure
|
|
297
|
+
begin
|
|
298
|
+
@beanstalk.connection.transmit("delete #{first}") if first
|
|
299
|
+
rescue Beaneater::NotFoundError
|
|
300
|
+
nil
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
it "buries the job when the parser raises an error process! handles itself" do
|
|
305
|
+
Beaneater.configure.job_parser = lambda do |body|
|
|
306
|
+
raise Beaneater::NotFoundError.new('NOT_FOUND', 'parse') if body == 'not found'
|
|
307
|
+
raise Beaneater::TimedOutError.new('TIMED_OUT', 'parse') if body == 'timed out'
|
|
308
|
+
body
|
|
309
|
+
end
|
|
310
|
+
handled = []
|
|
311
|
+
@jobs.register('tube_bad_body') do |job|
|
|
312
|
+
handled << job.body
|
|
313
|
+
@jobs.stop!
|
|
314
|
+
end
|
|
315
|
+
tube = @beanstalk.tubes.find('tube_bad_body')
|
|
316
|
+
not_found = tube.put('not found', :pri => 1, :ttr => 2)[:id]
|
|
317
|
+
timed_out = tube.put('timed out', :pri => 2, :ttr => 2)[:id]
|
|
318
|
+
tube.put('stop', :pri => 3)
|
|
319
|
+
|
|
320
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1) }
|
|
321
|
+
assert_equal ['stop'], handled
|
|
322
|
+
[not_found, timed_out].each do |id|
|
|
323
|
+
stats = stats_of(id)
|
|
324
|
+
assert_equal 'buried', stats['state'], "job #{id}"
|
|
325
|
+
assert_equal 0, stats['timeouts'], "job #{id}"
|
|
326
|
+
end
|
|
327
|
+
end
|
|
328
|
+
|
|
329
|
+
it "buries the job and stops when the parser raises AbortProcessingError" do
|
|
330
|
+
Beaneater.configure.job_parser = lambda { |body| raise Beaneater::AbortProcessingError if body == 'abort'; body }
|
|
331
|
+
@jobs.register('tube_bad_body') { |job| }
|
|
332
|
+
id = @beanstalk.tubes.find('tube_bad_body').put('abort')[:id]
|
|
333
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1) }
|
|
334
|
+
assert_equal 'buried', stats_of(id)['state']
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
it "buries the job when the parser raises a frozen exception" do
|
|
338
|
+
frozen = ArgumentError.new('frozen').freeze
|
|
339
|
+
Beaneater.configure.job_parser = lambda { |body| body == 'bad' ? raise(frozen) : body }
|
|
340
|
+
handled = []
|
|
341
|
+
@jobs.register('tube_bad_body') do |job|
|
|
342
|
+
handled << job.body
|
|
343
|
+
@jobs.stop!
|
|
344
|
+
end
|
|
345
|
+
tube = @beanstalk.tubes.find('tube_bad_body')
|
|
346
|
+
tube.put('bad', :pri => 1)
|
|
347
|
+
tube.put('good', :pri => 2)
|
|
348
|
+
|
|
349
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1) }
|
|
350
|
+
assert_equal ['good'], handled
|
|
351
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
352
|
+
assert_equal 'bad', (tube.peek(:buried) || flunk("the bad job was not buried")).body
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
it "buries the handler's job, not the one its own reserve rejected, when a handler raises it" do
|
|
356
|
+
cleanup_tubes!(['tube_bad_body_other'])
|
|
357
|
+
other_id = @beanstalk.tubes.find('tube_bad_body_other').put('not json')[:id]
|
|
358
|
+
@jobs.register('tube_bad_body') do |job|
|
|
359
|
+
@jobs.stop!
|
|
360
|
+
@beanstalk.tubes.watch('tube_bad_body_other')
|
|
361
|
+
@beanstalk.tubes.reserve(0) # raises: the parser rejects the other tube's job
|
|
362
|
+
end
|
|
363
|
+
@beanstalk.tubes.find('tube_bad_body').put('{"ok":1}')
|
|
364
|
+
|
|
365
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1) }
|
|
366
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
367
|
+
assert_equal '{"ok":1}', (@beanstalk.tubes.find('tube_bad_body').peek(:buried) || flunk("the handler's job was not buried")).body
|
|
368
|
+
assert_equal 'reserved', @beanstalk.connection.transmit("stats-job #{other_id}")[:body]['state']
|
|
369
|
+
ensure
|
|
370
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
371
|
+
begin
|
|
372
|
+
@beanstalk.connection.transmit("delete #{other_id}") if other_id
|
|
373
|
+
rescue Beaneater::NotFoundError
|
|
374
|
+
nil
|
|
375
|
+
end
|
|
376
|
+
cleanup_tubes!(['tube_bad_body_other'])
|
|
377
|
+
end
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
describe "for process! when a handler raises NotFoundError with its job reserved" do
|
|
381
|
+
before do
|
|
382
|
+
cleanup_tubes!(['tube_not_found'])
|
|
383
|
+
end
|
|
384
|
+
|
|
385
|
+
after do
|
|
386
|
+
cleanup_tubes!(['tube_not_found'])
|
|
387
|
+
end
|
|
388
|
+
|
|
389
|
+
it "buries the job once instead of reserving in a loop near its TTR" do
|
|
390
|
+
runs = 0
|
|
391
|
+
@jobs.register('tube_not_found') do |job|
|
|
392
|
+
runs += 1
|
|
393
|
+
@beanstalk.connection.transmit('delete 999999') # NOT_FOUND for some other job
|
|
394
|
+
end
|
|
395
|
+
id = @beanstalk.tubes.find('tube_not_found').put('x', :ttr => 2)[:id]
|
|
396
|
+
worker = Thread.new { @jobs.process!(:reserve_timeout => 1) }
|
|
397
|
+
sleep 3 # past the job's TTR
|
|
398
|
+
@jobs.stop!
|
|
399
|
+
assert worker.join(5), "process! did not stop"
|
|
400
|
+
checker = Beaneater.new('localhost')
|
|
401
|
+
stats = checker.connection.transmit("stats-job #{id}")[:body]
|
|
402
|
+
checker.close
|
|
403
|
+
assert_equal 1, runs
|
|
404
|
+
assert_equal 'buried', stats['state']
|
|
405
|
+
assert_equal 0, stats['timeouts']
|
|
406
|
+
end
|
|
407
|
+
|
|
408
|
+
it "goes on when the job stops being reserved just before it is buried" do
|
|
409
|
+
@jobs.register('tube_not_found') do |job|
|
|
410
|
+
@jobs.stop!
|
|
411
|
+
raise 'handler failed'
|
|
412
|
+
end
|
|
413
|
+
id = @beanstalk.tubes.find('tube_not_found').put('x')[:id]
|
|
414
|
+
Beaneater::Job.any_instance.stubs(:bury).raises(Beaneater::JobNotReserved) # another client's reservation ended
|
|
415
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1) } # must not raise JobNotReserved
|
|
416
|
+
assert_nil @jobs.current_job
|
|
417
|
+
ensure
|
|
418
|
+
Beaneater::Job.any_instance.unstub(:bury)
|
|
419
|
+
begin
|
|
420
|
+
@beanstalk.connection.transmit("delete #{id}") if id
|
|
421
|
+
rescue Beaneater::NotFoundError
|
|
422
|
+
nil
|
|
423
|
+
end
|
|
424
|
+
end
|
|
425
|
+
|
|
426
|
+
it "releases the job for another try instead when the error is in retry_on" do
|
|
427
|
+
runs = 0
|
|
428
|
+
@jobs.register('tube_not_found', :retry_on => [Beaneater::NotFoundError], :max_retries => 1) do |job|
|
|
429
|
+
runs += 1
|
|
430
|
+
@jobs.stop! if runs == 2
|
|
431
|
+
@beanstalk.connection.transmit('delete 999999') # NOT_FOUND for some other job
|
|
432
|
+
end
|
|
433
|
+
id = @beanstalk.tubes.find('tube_not_found').put('x')[:id]
|
|
434
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1, :release_delay => 0) }
|
|
435
|
+
stats = @beanstalk.connection.transmit("stats-job #{id}")[:body]
|
|
436
|
+
assert_equal 2, runs
|
|
437
|
+
assert_equal 1, stats['releases'] # released once, then buried once the retries ran out
|
|
438
|
+
assert_equal 'buried', stats['state']
|
|
439
|
+
end
|
|
440
|
+
end
|
|
441
|
+
|
|
442
|
+
describe "for process! when a handler raises NotConnected for another server" do
|
|
443
|
+
before do
|
|
444
|
+
cleanup_tubes!(['tube_other_down'])
|
|
445
|
+
end
|
|
446
|
+
|
|
447
|
+
it "buries the job and keeps processing" do
|
|
448
|
+
seen = []
|
|
449
|
+
@jobs.register('tube_other_down') do |job|
|
|
450
|
+
seen << job.body
|
|
451
|
+
@jobs.stop! if job.body == 'second'
|
|
452
|
+
raise Beaneater::NotConnected, "the other server is down" if job.body == 'first'
|
|
453
|
+
end
|
|
454
|
+
tube = @beanstalk.tubes.find('tube_other_down')
|
|
455
|
+
tube.put('first', :pri => 1)
|
|
456
|
+
tube.put('second', :pri => 2)
|
|
457
|
+
|
|
458
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1) }
|
|
459
|
+
assert_equal ['first', 'second'], seen
|
|
460
|
+
assert_equal 'first', tube.peek(:buried).body
|
|
461
|
+
end
|
|
462
|
+
end
|
|
463
|
+
|
|
464
|
+
describe "for process! when the server goes away" do
|
|
465
|
+
before do
|
|
466
|
+
@port = free_port
|
|
467
|
+
@pid = start_beanstalkd(@port)
|
|
468
|
+
@client = Beaneater.new("127.0.0.1:#{@port}")
|
|
469
|
+
end
|
|
470
|
+
|
|
471
|
+
after do
|
|
472
|
+
@client.close if @client
|
|
473
|
+
stop_beanstalkd(@pid) if @pid
|
|
474
|
+
end
|
|
475
|
+
|
|
476
|
+
it "forgets the job when burying it fails, so the next run buries its own bad job" do
|
|
477
|
+
Beaneater.configure.job_parser = lambda do |body|
|
|
478
|
+
stop_beanstalkd(@pid) if body == 'kill' # the server dies before the bad job is buried
|
|
479
|
+
JSON.parse(body)
|
|
480
|
+
end
|
|
481
|
+
handled = []
|
|
482
|
+
@client.jobs.register('bury_fails') do |job|
|
|
483
|
+
handled << job.body
|
|
484
|
+
@client.jobs.stop!
|
|
485
|
+
end
|
|
486
|
+
@client.tubes.find('bury_fails').put('kill')
|
|
487
|
+
assert_raises(Beaneater::NotConnected) { Timeout.timeout(10) { @client.jobs.process!(:reserve_timeout => 1) } }
|
|
488
|
+
assert_nil @client.jobs.current_job
|
|
489
|
+
|
|
490
|
+
@pid = start_beanstalkd(@port) # a new server: job ids start again at 1
|
|
491
|
+
producer = Beaneater.new("127.0.0.1:#{@port}")
|
|
492
|
+
producer.tubes.find('bury_fails_filler').put('filler') # takes the id the lost job had
|
|
493
|
+
producer.tubes.find('bury_fails').put('not json', :pri => 1)
|
|
494
|
+
producer.tubes.find('bury_fails').put('{"ok":1}', :pri => 2)
|
|
495
|
+
Timeout.timeout(10) { @client.jobs.process!(:reserve_timeout => 1) }
|
|
496
|
+
assert_equal [{ 'ok' => 1 }], handled
|
|
497
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
498
|
+
buried = producer.tubes.find('bury_fails').peek(:buried)
|
|
499
|
+
refute_nil buried, "the bad job was not buried"
|
|
500
|
+
assert_equal 'not json', buried.body
|
|
501
|
+
producer.close
|
|
502
|
+
ensure
|
|
503
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
504
|
+
end
|
|
505
|
+
|
|
506
|
+
it "raises NotConnected instead of looping, and the client resumes once the server is back" do
|
|
507
|
+
seen = []
|
|
508
|
+
@client.jobs.register('reconnect') do |job|
|
|
509
|
+
seen << job.body
|
|
510
|
+
@client.jobs.stop! if job.body == 'after restart'
|
|
511
|
+
end
|
|
512
|
+
@client.tubes.find('reconnect').put('before outage')
|
|
513
|
+
|
|
514
|
+
worker = Thread.new do
|
|
515
|
+
begin
|
|
516
|
+
@client.jobs.process!(:reserve_timeout => 1)
|
|
517
|
+
rescue StandardError => ex
|
|
518
|
+
ex
|
|
519
|
+
end
|
|
520
|
+
end
|
|
521
|
+
Timeout.timeout(5) { sleep 0.05 while seen.empty? }
|
|
522
|
+
stop_beanstalkd(@pid)
|
|
523
|
+
|
|
524
|
+
finished = worker.join(10)
|
|
525
|
+
worker.kill unless finished
|
|
526
|
+
assert finished, "process! was still running 10s after the server went away"
|
|
527
|
+
assert_kind_of Beaneater::NotConnected, worker.value
|
|
528
|
+
|
|
529
|
+
@pid = start_beanstalkd(@port)
|
|
530
|
+
producer = Beaneater.new("127.0.0.1:#{@port}")
|
|
531
|
+
producer.tubes.find('reconnect').put('after restart')
|
|
532
|
+
producer.close
|
|
533
|
+
|
|
534
|
+
Timeout.timeout(10) { @client.jobs.process!(:reserve_timeout => 1) }
|
|
535
|
+
assert_equal ['before outage', 'after restart'], seen
|
|
536
|
+
end
|
|
537
|
+
end
|
|
107
538
|
end # Beaneater::Jobs
|
data/test/prompt_regexp_test.rb
CHANGED
data/test/stat_struct_test.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
# test/stat_struct_test.rb
|
|
2
4
|
|
|
3
5
|
require File.expand_path('../test_helper', __FILE__)
|
|
@@ -27,6 +29,11 @@ describe Beaneater::StatStruct do
|
|
|
27
29
|
assert_equal 'foo', @struct[:baz]
|
|
28
30
|
assert_equal 'demo', @struct[:"under_score"]
|
|
29
31
|
end
|
|
32
|
+
|
|
33
|
+
it "should look up hyphenated beanstalkd keys" do
|
|
34
|
+
assert_equal 'demo', @struct['under-score']
|
|
35
|
+
assert_equal 'demo', @struct[:"under-score"]
|
|
36
|
+
end
|
|
30
37
|
end # []
|
|
31
38
|
|
|
32
39
|
describe "for #keys" do
|
data/test/stats_test.rb
CHANGED
data/test/test_helper.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
ENV["TEST"] = 'true'
|
|
2
4
|
require 'rubygems'
|
|
3
5
|
require 'coveralls'
|
|
@@ -43,4 +45,49 @@ class MiniTest::Unit::TestCase
|
|
|
43
45
|
def teardown
|
|
44
46
|
flush_all
|
|
45
47
|
end
|
|
48
|
+
|
|
49
|
+
# The beanstalkd binary for tests that stop and restart their own server: BEANSTALKD_BIN, or
|
|
50
|
+
# `beanstalkd` on PATH. Skips the test when neither is available.
|
|
51
|
+
def beanstalkd_bin
|
|
52
|
+
bin = ENV['BEANSTALKD_BIN']
|
|
53
|
+
return bin if bin && !bin.empty?
|
|
54
|
+
on_path = ENV['PATH'].to_s.split(File::PATH_SEPARATOR).any? do |dir|
|
|
55
|
+
File.executable?(File.join(dir, 'beanstalkd'))
|
|
56
|
+
end
|
|
57
|
+
skip("beanstalkd not found: put it on PATH or set BEANSTALKD_BIN") unless on_path
|
|
58
|
+
'beanstalkd'
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# A free local port for a private beanstalkd
|
|
62
|
+
def free_port
|
|
63
|
+
server = TCPServer.new('127.0.0.1', 0)
|
|
64
|
+
server.addr[1]
|
|
65
|
+
ensure
|
|
66
|
+
server.close if server
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Starts a private beanstalkd on port and waits until it accepts connections.
|
|
70
|
+
#
|
|
71
|
+
# @return [Integer] the server's pid
|
|
72
|
+
def start_beanstalkd(port)
|
|
73
|
+
pid = Process.spawn(beanstalkd_bin, '-l', '127.0.0.1', '-p', port.to_s, [:out, :err] => File::NULL)
|
|
74
|
+
50.times do
|
|
75
|
+
begin
|
|
76
|
+
TCPSocket.new('127.0.0.1', port).close
|
|
77
|
+
return pid
|
|
78
|
+
rescue Errno::ECONNREFUSED
|
|
79
|
+
sleep 0.05
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
stop_beanstalkd(pid)
|
|
83
|
+
flunk "beanstalkd did not start on port #{port}"
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Kills a server started by start_beanstalkd
|
|
87
|
+
def stop_beanstalkd(pid)
|
|
88
|
+
Process.kill('KILL', pid)
|
|
89
|
+
Process.wait(pid)
|
|
90
|
+
rescue Errno::ESRCH, Errno::ECHILD
|
|
91
|
+
nil
|
|
92
|
+
end
|
|
46
93
|
end
|
data/test/tube_test.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
# test/tube_test.rb
|
|
2
4
|
|
|
3
5
|
require File.expand_path('../test_helper', __FILE__)
|
|
@@ -94,11 +96,134 @@ describe Beaneater::Tube do
|
|
|
94
96
|
assert_equal 'hi: there! this: is a long message', @tube.peek(:ready).body['message']
|
|
95
97
|
end
|
|
96
98
|
|
|
99
|
+
it "should keep the connection usable when the custom parser raises" do
|
|
100
|
+
Beaneater.configure.job_parser = lambda { |b| JSON.parse(b) }
|
|
101
|
+
@tube.put('not json')
|
|
102
|
+
socket = @beanstalk.connection.connection
|
|
103
|
+
assert_raises(JSON::ParserError) { @tube.peek(:ready) }
|
|
104
|
+
assert_same socket, @beanstalk.connection.connection
|
|
105
|
+
assert_equal 'baz', @beanstalk.tubes.used.name
|
|
106
|
+
end
|
|
107
|
+
|
|
97
108
|
after do
|
|
98
109
|
Beaneater.configure.job_parser = lambda { |b| b }
|
|
99
110
|
end
|
|
100
111
|
end # peek
|
|
101
112
|
|
|
113
|
+
describe "for a job_parser that raises on reserve" do
|
|
114
|
+
before do
|
|
115
|
+
Beaneater.configure.job_parser = lambda { |b| JSON.parse(b) }
|
|
116
|
+
@tube = @beanstalk.tubes.find('poison_tube')
|
|
117
|
+
@poison_id = @tube.put('not json', :pri => 1)[:id]
|
|
118
|
+
@tube.put('{"ok":1}', :pri => 2)
|
|
119
|
+
@beanstalk.tubes.watch!('poison_tube')
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
after do
|
|
123
|
+
Beaneater.configure.job_parser = lambda { |b| b }
|
|
124
|
+
# The bad job is still reserved by this connection, which clearing the tube can't touch;
|
|
125
|
+
# left alone, it would be released when the socket is collected, into a later test.
|
|
126
|
+
begin
|
|
127
|
+
@beanstalk.connection.transmit("delete #{@poison_id}")
|
|
128
|
+
rescue Beaneater::NotFoundError
|
|
129
|
+
nil
|
|
130
|
+
end
|
|
131
|
+
cleanup_tubes!(['poison_tube'])
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
it "should raise the parser's own error from reserve, tagged with the job id" do
|
|
135
|
+
error = assert_raises(JSON::ParserError) { @beanstalk.tubes.reserve(0) }
|
|
136
|
+
assert_kind_of Beaneater::JobParseFailure, error
|
|
137
|
+
assert_equal @poison_id, error.job_id
|
|
138
|
+
assert_nil error.cause # the parser's error had none
|
|
139
|
+
Beaneater.configure.job_parser = lambda { |b| b }
|
|
140
|
+
assert_equal 'reserved', @beanstalk.connection.transmit("stats-job #{error.job_id}")[:body]['state']
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
it "should keep the parser's error intact: class, message, backtrace and cause" do
|
|
144
|
+
original = nil
|
|
145
|
+
Beaneater.configure.job_parser = lambda do |b|
|
|
146
|
+
begin
|
|
147
|
+
JSON.parse(b)
|
|
148
|
+
rescue JSON::ParserError => ex
|
|
149
|
+
original = ex
|
|
150
|
+
raise ArgumentError, 'not a job' # its cause is the ParserError
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
error = assert_raises(ArgumentError) { @beanstalk.tubes.reserve(0) }
|
|
154
|
+
assert_equal @poison_id, error.job_id
|
|
155
|
+
assert_equal 'not a job', error.message
|
|
156
|
+
assert_same original, error.cause
|
|
157
|
+
assert_operator error.backtrace.first, :include?, 'tube_test.rb'
|
|
158
|
+
refute_nil error.backtrace_locations
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# An exception class that can't be copied on any Ruby (clone calls initialize_copy)
|
|
162
|
+
class UncopyableError < StandardError
|
|
163
|
+
def initialize_copy(_original)
|
|
164
|
+
raise TypeError, "can't copy"
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# An exception class whose #cause is not an exception
|
|
169
|
+
class OddCauseError < StandardError
|
|
170
|
+
def cause
|
|
171
|
+
'not an exception'
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
it "should raise the parser's own exception, untagged, when it can't be copied" do
|
|
176
|
+
Beaneater.configure.job_parser = lambda { |b| raise UncopyableError, 'no copies' }
|
|
177
|
+
error = assert_raises(UncopyableError) { @beanstalk.tubes.reserve(0) }
|
|
178
|
+
assert_equal 'no copies', error.message
|
|
179
|
+
refute_kind_of Beaneater::JobParseFailure, error
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
it "should raise a frozen exception that can't be copied as it is" do
|
|
183
|
+
frozen = UncopyableError.new('frozen, no copies').freeze
|
|
184
|
+
Beaneater.configure.job_parser = lambda { |b| raise frozen }
|
|
185
|
+
error = assert_raises(UncopyableError) { @beanstalk.tubes.reserve(0) }
|
|
186
|
+
assert_same frozen, error
|
|
187
|
+
assert_nil error.cause
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
it "should tag the copy when the exception's cause is not an exception" do
|
|
191
|
+
Beaneater.configure.job_parser = lambda { |b| raise OddCauseError, 'odd' }
|
|
192
|
+
error = assert_raises(OddCauseError) { @beanstalk.tubes.reserve(0) }
|
|
193
|
+
assert_equal @poison_id, error.job_id
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
it "should not tag an exception the parser reuses, so a later error carries no old id" do
|
|
197
|
+
shared = JSON::ParserError.new('always this one')
|
|
198
|
+
Beaneater.configure.job_parser = lambda { |b| raise shared }
|
|
199
|
+
error = assert_raises(JSON::ParserError) { @beanstalk.tubes.reserve(0) }
|
|
200
|
+
assert_equal @poison_id, error.job_id
|
|
201
|
+
peeked = assert_raises(JSON::ParserError) { @tube.peek(:ready) } # not a reserve: not tagged
|
|
202
|
+
refute_kind_of Beaneater::JobParseFailure, peeked
|
|
203
|
+
refute_kind_of Beaneater::JobParseFailure, shared
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
it "should keep the connection, so the bad job stays reserved and the next job is delivered" do
|
|
207
|
+
socket = @beanstalk.connection.connection
|
|
208
|
+
assert_raises(JSON::ParserError) { @beanstalk.tubes.reserve(0) }
|
|
209
|
+
assert_same socket, @beanstalk.connection.connection
|
|
210
|
+
job = @beanstalk.tubes.reserve(0)
|
|
211
|
+
assert_equal({ 'ok' => 1 }, job.body)
|
|
212
|
+
job.delete
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
it "should keep the connection when the raising parser used the same client first" do
|
|
216
|
+
tube = 'poison_tube'
|
|
217
|
+
Beaneater.configure.job_parser = lambda { |b| @beanstalk.connection.transmit("use #{tube}"); JSON.parse(b) }
|
|
218
|
+
socket = @beanstalk.connection.connection
|
|
219
|
+
assert_raises(JSON::ParserError) { @beanstalk.tubes.reserve(0) }
|
|
220
|
+
assert_same socket, @beanstalk.connection.connection
|
|
221
|
+
job = @beanstalk.tubes.reserve(0)
|
|
222
|
+
assert_equal({ 'ok' => 1 }, job.body)
|
|
223
|
+
job.delete
|
|
224
|
+
end
|
|
225
|
+
end # job_parser that raises on reserve
|
|
226
|
+
|
|
102
227
|
describe "for #reserve" do
|
|
103
228
|
before do
|
|
104
229
|
@time = Time.now.to_i
|