beaneater 1.1.4 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +4 -2
- data/CHANGELOG.md +16 -0
- data/README.md +28 -0
- data/beaneater.gemspec +1 -0
- data/lib/beaneater/configuration.rb +8 -0
- data/lib/beaneater/connection.rb +313 -57
- data/lib/beaneater/errors.rb +14 -0
- data/lib/beaneater/job/collection.rb +42 -6
- data/lib/beaneater/stats/stat_struct.rb +6 -1
- data/lib/beaneater/version.rb +1 -1
- data/test/connection_test.rb +610 -15
- data/test/job_test.rb +11 -0
- data/test/jobs_test.rb +429 -0
- data/test/stat_struct_test.rb +5 -0
- data/test/test_helper.rb +45 -0
- data/test/tube_test.rb +123 -0
- data/test/tubes_test.rb +16 -1
- metadata +3 -3
data/test/jobs_test.rb
CHANGED
|
@@ -106,4 +106,433 @@ describe Beaneater::Jobs do
|
|
|
106
106
|
assert_equal 1, @beanstalk.tubes.find('tube_buried').stats.current_jobs_buried
|
|
107
107
|
end
|
|
108
108
|
end # for_process!
|
|
109
|
+
|
|
110
|
+
describe "for process! with a dashed tube name" do
|
|
111
|
+
before do
|
|
112
|
+
$ran = []
|
|
113
|
+
cleanup_tubes!(['email-jobs', 'tube_stop'])
|
|
114
|
+
@jobs.register('email-jobs') { |job| $ran << job.body }
|
|
115
|
+
@jobs.register('tube_stop') { |job| $ran << job.body; @jobs.stop! }
|
|
116
|
+
@beanstalk.tubes.find('email-jobs').put('email', :pri => 1)
|
|
117
|
+
@beanstalk.tubes.find('tube_stop').put('stop', :pri => 2)
|
|
118
|
+
@jobs.process!(:release_delay => 0)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
it "should run the handler registered for the tube" do
|
|
122
|
+
assert_equal ['email', 'stop'], $ran
|
|
123
|
+
assert_nil @beanstalk.tubes.find('email-jobs').peek(:buried)
|
|
124
|
+
end
|
|
125
|
+
end # process! with a dashed tube name
|
|
126
|
+
|
|
127
|
+
describe "for process! when a failing handler no longer holds its job" do
|
|
128
|
+
before do
|
|
129
|
+
$foo = []
|
|
130
|
+
cleanup_tubes!(['tube_lost'])
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
after do
|
|
134
|
+
cleanup_tubes!(['tube_lost'])
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# The handler loses 'first', puts 'second' and raises. 'second' outranks 'first', so it is
|
|
138
|
+
# reserved next whatever happened to 'first'.
|
|
139
|
+
def process_until_second_job(ttr = 120, &lose_job)
|
|
140
|
+
tube = @beanstalk.tubes.find('tube_lost')
|
|
141
|
+
@jobs.register('tube_lost') do |job|
|
|
142
|
+
$foo << job.body
|
|
143
|
+
if job.body == 'second'
|
|
144
|
+
@jobs.stop!
|
|
145
|
+
else
|
|
146
|
+
lose_job.call(job)
|
|
147
|
+
tube.put('second', :pri => 0)
|
|
148
|
+
raise RuntimeError
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
tube.put('first', :pri => 1, :ttr => ttr)
|
|
152
|
+
Timeout.timeout(15) { @jobs.process!(:reserve_timeout => 1) }
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
it "keeps processing when the handler deleted the job" do
|
|
156
|
+
process_until_second_job { |job| job.delete }
|
|
157
|
+
assert_equal ['first', 'second'], $foo
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
it "keeps processing when the handler released the job" do
|
|
161
|
+
process_until_second_job { |job| job.release(:delay => 5) }
|
|
162
|
+
assert_equal ['first', 'second'], $foo
|
|
163
|
+
assert_equal 1, @beanstalk.tubes.find('tube_lost').stats.current_jobs_delayed
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
it "keeps processing when the handler overran the job's TTR" do
|
|
167
|
+
process_until_second_job(1) do |job|
|
|
168
|
+
# the server releases a job some time after its TTR runs out
|
|
169
|
+
deadline = Time.now + 10
|
|
170
|
+
sleep 0.1 while job.stats.state == 'reserved' && Time.now < deadline
|
|
171
|
+
refute_equal 'reserved', job.stats.state
|
|
172
|
+
end
|
|
173
|
+
assert_equal ['first', 'second'], $foo
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
describe "for process! with stop!" do
|
|
178
|
+
before do
|
|
179
|
+
cleanup_tubes!(['tube_stop'])
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
after do
|
|
183
|
+
cleanup_tubes!(['tube_stop'])
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# Runs process! in a thread and yields. Returns :returned, the exception process! raised, or
|
|
187
|
+
# :still_running if it hadn't finished within seconds.
|
|
188
|
+
def process_in_thread(seconds)
|
|
189
|
+
worker = Thread.new do
|
|
190
|
+
begin
|
|
191
|
+
@jobs.process!(:reserve_timeout => 1)
|
|
192
|
+
:returned
|
|
193
|
+
rescue StandardError => ex
|
|
194
|
+
ex
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
yield if block_given?
|
|
198
|
+
return worker.value if worker.join(seconds)
|
|
199
|
+
worker.kill
|
|
200
|
+
:still_running
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
it "returns within one reserve_timeout when stop! is called while idle" do
|
|
204
|
+
@jobs.register('tube_stop') { |job| }
|
|
205
|
+
result = process_in_thread(3) do
|
|
206
|
+
sleep 0.5
|
|
207
|
+
@jobs.stop!
|
|
208
|
+
end
|
|
209
|
+
assert_equal :returned, result
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
it "returns when stop! is called by a handler whose job another worker took over" do
|
|
213
|
+
other = Beaneater.new('localhost')
|
|
214
|
+
other.tubes.watch!('tube_stop')
|
|
215
|
+
runs = 0
|
|
216
|
+
@jobs.register('tube_stop') do |job|
|
|
217
|
+
runs += 1
|
|
218
|
+
job.release # as the server does once a job's TTR runs out
|
|
219
|
+
other.tubes.reserve(1)
|
|
220
|
+
@jobs.stop!
|
|
221
|
+
end
|
|
222
|
+
@beanstalk.tubes.find('tube_stop').put('taken over')
|
|
223
|
+
|
|
224
|
+
assert_equal :returned, process_in_thread(5)
|
|
225
|
+
assert_equal 1, runs
|
|
226
|
+
ensure
|
|
227
|
+
other.close if other
|
|
228
|
+
end
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
describe "for process! when the job_parser rejects a job" do
|
|
232
|
+
before do
|
|
233
|
+
cleanup_tubes!(['tube_bad_body'])
|
|
234
|
+
Beaneater.configure.job_parser = lambda { |body| JSON.parse(body) }
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
after do
|
|
238
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
239
|
+
cleanup_tubes!(['tube_bad_body'])
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
it "buries the job and goes on to the next one" do
|
|
243
|
+
handled = []
|
|
244
|
+
@jobs.register('tube_bad_body') do |job|
|
|
245
|
+
handled << job.body
|
|
246
|
+
@jobs.stop!
|
|
247
|
+
end
|
|
248
|
+
tube = @beanstalk.tubes.find('tube_bad_body')
|
|
249
|
+
tube.put('not json', :pri => 1)
|
|
250
|
+
tube.put('{"ok":1}', :pri => 2)
|
|
251
|
+
|
|
252
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1) }
|
|
253
|
+
assert_equal [{ 'ok' => 1 }], handled
|
|
254
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
255
|
+
buried = tube.peek(:buried)
|
|
256
|
+
refute_nil buried, "the job the parser rejected was not buried"
|
|
257
|
+
assert_equal 'not json', buried.body
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
# process! goes on after NotFoundError (and JobNotReserved, TimedOutError); the job it had
|
|
261
|
+
# must not be taken for the bad job that follows.
|
|
262
|
+
def put_after_retry_jobs
|
|
263
|
+
tube = @beanstalk.tubes.find('tube_bad_body')
|
|
264
|
+
first = tube.put('{"first":1}', :pri => 1)[:id]
|
|
265
|
+
bad = tube.put('not json', :pri => 2)[:id]
|
|
266
|
+
tube.put('{"stop":1}', :pri => 3)
|
|
267
|
+
[tube, first, bad]
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
def stats_of(id)
|
|
271
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
272
|
+
@beanstalk.connection.transmit("stats-job #{id}")[:body]
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
it "buries the bad job after a handler deleted its own job" do
|
|
276
|
+
@jobs.register('tube_bad_body') do |job|
|
|
277
|
+
job.body == { 'stop' => 1 } ? @jobs.stop! : job.delete # process!'s own delete then gets NOT_FOUND
|
|
278
|
+
end
|
|
279
|
+
tube, _first, bad = put_after_retry_jobs
|
|
280
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1) }
|
|
281
|
+
assert_equal 'buried', stats_of(bad)['state']
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
it "buries both the failed job and the bad job after a handler raised NotFoundError" do
|
|
285
|
+
@jobs.register('tube_bad_body') do |job|
|
|
286
|
+
if job.body == { 'stop' => 1 }
|
|
287
|
+
@jobs.stop!
|
|
288
|
+
else
|
|
289
|
+
@beanstalk.connection.transmit('delete 999999') # NOT_FOUND for some other job
|
|
290
|
+
end
|
|
291
|
+
end
|
|
292
|
+
tube, first, bad = put_after_retry_jobs
|
|
293
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1) }
|
|
294
|
+
assert_equal 'buried', stats_of(bad)['state']
|
|
295
|
+
assert_equal 'buried', stats_of(first)['state'] # its handler failed, with the job still reserved
|
|
296
|
+
ensure
|
|
297
|
+
begin
|
|
298
|
+
@beanstalk.connection.transmit("delete #{first}") if first
|
|
299
|
+
rescue Beaneater::NotFoundError
|
|
300
|
+
nil
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
it "buries the job when the parser raises an error process! handles itself" do
|
|
305
|
+
Beaneater.configure.job_parser = lambda do |body|
|
|
306
|
+
raise Beaneater::NotFoundError.new('NOT_FOUND', 'parse') if body == 'not found'
|
|
307
|
+
raise Beaneater::TimedOutError.new('TIMED_OUT', 'parse') if body == 'timed out'
|
|
308
|
+
body
|
|
309
|
+
end
|
|
310
|
+
handled = []
|
|
311
|
+
@jobs.register('tube_bad_body') do |job|
|
|
312
|
+
handled << job.body
|
|
313
|
+
@jobs.stop!
|
|
314
|
+
end
|
|
315
|
+
tube = @beanstalk.tubes.find('tube_bad_body')
|
|
316
|
+
not_found = tube.put('not found', :pri => 1, :ttr => 2)[:id]
|
|
317
|
+
timed_out = tube.put('timed out', :pri => 2, :ttr => 2)[:id]
|
|
318
|
+
tube.put('stop', :pri => 3)
|
|
319
|
+
|
|
320
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1) }
|
|
321
|
+
assert_equal ['stop'], handled
|
|
322
|
+
[not_found, timed_out].each do |id|
|
|
323
|
+
stats = stats_of(id)
|
|
324
|
+
assert_equal 'buried', stats['state'], "job #{id}"
|
|
325
|
+
assert_equal 0, stats['timeouts'], "job #{id}"
|
|
326
|
+
end
|
|
327
|
+
end
|
|
328
|
+
|
|
329
|
+
it "buries the job and stops when the parser raises AbortProcessingError" do
|
|
330
|
+
Beaneater.configure.job_parser = lambda { |body| raise Beaneater::AbortProcessingError if body == 'abort'; body }
|
|
331
|
+
@jobs.register('tube_bad_body') { |job| }
|
|
332
|
+
id = @beanstalk.tubes.find('tube_bad_body').put('abort')[:id]
|
|
333
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1) }
|
|
334
|
+
assert_equal 'buried', stats_of(id)['state']
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
it "buries the job when the parser raises a frozen exception" do
|
|
338
|
+
frozen = ArgumentError.new('frozen').freeze
|
|
339
|
+
Beaneater.configure.job_parser = lambda { |body| body == 'bad' ? raise(frozen) : body }
|
|
340
|
+
handled = []
|
|
341
|
+
@jobs.register('tube_bad_body') do |job|
|
|
342
|
+
handled << job.body
|
|
343
|
+
@jobs.stop!
|
|
344
|
+
end
|
|
345
|
+
tube = @beanstalk.tubes.find('tube_bad_body')
|
|
346
|
+
tube.put('bad', :pri => 1)
|
|
347
|
+
tube.put('good', :pri => 2)
|
|
348
|
+
|
|
349
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1) }
|
|
350
|
+
assert_equal ['good'], handled
|
|
351
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
352
|
+
assert_equal 'bad', (tube.peek(:buried) || flunk("the bad job was not buried")).body
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
it "buries the handler's job, not the one its own reserve rejected, when a handler raises it" do
|
|
356
|
+
cleanup_tubes!(['tube_bad_body_other'])
|
|
357
|
+
other_id = @beanstalk.tubes.find('tube_bad_body_other').put('not json')[:id]
|
|
358
|
+
@jobs.register('tube_bad_body') do |job|
|
|
359
|
+
@jobs.stop!
|
|
360
|
+
@beanstalk.tubes.watch('tube_bad_body_other')
|
|
361
|
+
@beanstalk.tubes.reserve(0) # raises: the parser rejects the other tube's job
|
|
362
|
+
end
|
|
363
|
+
@beanstalk.tubes.find('tube_bad_body').put('{"ok":1}')
|
|
364
|
+
|
|
365
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1) }
|
|
366
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
367
|
+
assert_equal '{"ok":1}', (@beanstalk.tubes.find('tube_bad_body').peek(:buried) || flunk("the handler's job was not buried")).body
|
|
368
|
+
assert_equal 'reserved', @beanstalk.connection.transmit("stats-job #{other_id}")[:body]['state']
|
|
369
|
+
ensure
|
|
370
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
371
|
+
begin
|
|
372
|
+
@beanstalk.connection.transmit("delete #{other_id}") if other_id
|
|
373
|
+
rescue Beaneater::NotFoundError
|
|
374
|
+
nil
|
|
375
|
+
end
|
|
376
|
+
cleanup_tubes!(['tube_bad_body_other'])
|
|
377
|
+
end
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
describe "for process! when a handler raises NotFoundError with its job reserved" do
|
|
381
|
+
before do
|
|
382
|
+
cleanup_tubes!(['tube_not_found'])
|
|
383
|
+
end
|
|
384
|
+
|
|
385
|
+
after do
|
|
386
|
+
cleanup_tubes!(['tube_not_found'])
|
|
387
|
+
end
|
|
388
|
+
|
|
389
|
+
it "buries the job once instead of reserving in a loop near its TTR" do
|
|
390
|
+
runs = 0
|
|
391
|
+
@jobs.register('tube_not_found') do |job|
|
|
392
|
+
runs += 1
|
|
393
|
+
@beanstalk.connection.transmit('delete 999999') # NOT_FOUND for some other job
|
|
394
|
+
end
|
|
395
|
+
id = @beanstalk.tubes.find('tube_not_found').put('x', :ttr => 2)[:id]
|
|
396
|
+
worker = Thread.new { @jobs.process!(:reserve_timeout => 1) }
|
|
397
|
+
sleep 3 # past the job's TTR
|
|
398
|
+
@jobs.stop!
|
|
399
|
+
assert worker.join(5), "process! did not stop"
|
|
400
|
+
checker = Beaneater.new('localhost')
|
|
401
|
+
stats = checker.connection.transmit("stats-job #{id}")[:body]
|
|
402
|
+
checker.close
|
|
403
|
+
assert_equal 1, runs
|
|
404
|
+
assert_equal 'buried', stats['state']
|
|
405
|
+
assert_equal 0, stats['timeouts']
|
|
406
|
+
end
|
|
407
|
+
|
|
408
|
+
it "goes on when the job stops being reserved just before it is buried" do
|
|
409
|
+
@jobs.register('tube_not_found') do |job|
|
|
410
|
+
@jobs.stop!
|
|
411
|
+
raise 'handler failed'
|
|
412
|
+
end
|
|
413
|
+
id = @beanstalk.tubes.find('tube_not_found').put('x')[:id]
|
|
414
|
+
Beaneater::Job.any_instance.stubs(:bury).raises(Beaneater::JobNotReserved) # another client's reservation ended
|
|
415
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1) } # must not raise JobNotReserved
|
|
416
|
+
assert_nil @jobs.current_job
|
|
417
|
+
ensure
|
|
418
|
+
Beaneater::Job.any_instance.unstub(:bury)
|
|
419
|
+
begin
|
|
420
|
+
@beanstalk.connection.transmit("delete #{id}") if id
|
|
421
|
+
rescue Beaneater::NotFoundError
|
|
422
|
+
nil
|
|
423
|
+
end
|
|
424
|
+
end
|
|
425
|
+
|
|
426
|
+
it "releases the job for another try instead when the error is in retry_on" do
|
|
427
|
+
runs = 0
|
|
428
|
+
@jobs.register('tube_not_found', :retry_on => [Beaneater::NotFoundError], :max_retries => 1) do |job|
|
|
429
|
+
runs += 1
|
|
430
|
+
@jobs.stop! if runs == 2
|
|
431
|
+
@beanstalk.connection.transmit('delete 999999') # NOT_FOUND for some other job
|
|
432
|
+
end
|
|
433
|
+
id = @beanstalk.tubes.find('tube_not_found').put('x')[:id]
|
|
434
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1, :release_delay => 0) }
|
|
435
|
+
stats = @beanstalk.connection.transmit("stats-job #{id}")[:body]
|
|
436
|
+
assert_equal 2, runs
|
|
437
|
+
assert_equal 1, stats['releases'] # released once, then buried once the retries ran out
|
|
438
|
+
assert_equal 'buried', stats['state']
|
|
439
|
+
end
|
|
440
|
+
end
|
|
441
|
+
|
|
442
|
+
describe "for process! when a handler raises NotConnected for another server" do
|
|
443
|
+
before do
|
|
444
|
+
cleanup_tubes!(['tube_other_down'])
|
|
445
|
+
end
|
|
446
|
+
|
|
447
|
+
it "buries the job and keeps processing" do
|
|
448
|
+
seen = []
|
|
449
|
+
@jobs.register('tube_other_down') do |job|
|
|
450
|
+
seen << job.body
|
|
451
|
+
@jobs.stop! if job.body == 'second'
|
|
452
|
+
raise Beaneater::NotConnected, "the other server is down" if job.body == 'first'
|
|
453
|
+
end
|
|
454
|
+
tube = @beanstalk.tubes.find('tube_other_down')
|
|
455
|
+
tube.put('first', :pri => 1)
|
|
456
|
+
tube.put('second', :pri => 2)
|
|
457
|
+
|
|
458
|
+
Timeout.timeout(10) { @jobs.process!(:reserve_timeout => 1) }
|
|
459
|
+
assert_equal ['first', 'second'], seen
|
|
460
|
+
assert_equal 'first', tube.peek(:buried).body
|
|
461
|
+
end
|
|
462
|
+
end
|
|
463
|
+
|
|
464
|
+
describe "for process! when the server goes away" do
|
|
465
|
+
before do
|
|
466
|
+
@port = free_port
|
|
467
|
+
@pid = start_beanstalkd(@port)
|
|
468
|
+
@client = Beaneater.new("127.0.0.1:#{@port}")
|
|
469
|
+
end
|
|
470
|
+
|
|
471
|
+
after do
|
|
472
|
+
@client.close if @client
|
|
473
|
+
stop_beanstalkd(@pid) if @pid
|
|
474
|
+
end
|
|
475
|
+
|
|
476
|
+
it "forgets the job when burying it fails, so the next run buries its own bad job" do
|
|
477
|
+
Beaneater.configure.job_parser = lambda do |body|
|
|
478
|
+
stop_beanstalkd(@pid) if body == 'kill' # the server dies before the bad job is buried
|
|
479
|
+
JSON.parse(body)
|
|
480
|
+
end
|
|
481
|
+
handled = []
|
|
482
|
+
@client.jobs.register('bury_fails') do |job|
|
|
483
|
+
handled << job.body
|
|
484
|
+
@client.jobs.stop!
|
|
485
|
+
end
|
|
486
|
+
@client.tubes.find('bury_fails').put('kill')
|
|
487
|
+
assert_raises(Beaneater::NotConnected) { Timeout.timeout(10) { @client.jobs.process!(:reserve_timeout => 1) } }
|
|
488
|
+
assert_nil @client.jobs.current_job
|
|
489
|
+
|
|
490
|
+
@pid = start_beanstalkd(@port) # a new server: job ids start again at 1
|
|
491
|
+
producer = Beaneater.new("127.0.0.1:#{@port}")
|
|
492
|
+
producer.tubes.find('bury_fails_filler').put('filler') # takes the id the lost job had
|
|
493
|
+
producer.tubes.find('bury_fails').put('not json', :pri => 1)
|
|
494
|
+
producer.tubes.find('bury_fails').put('{"ok":1}', :pri => 2)
|
|
495
|
+
Timeout.timeout(10) { @client.jobs.process!(:reserve_timeout => 1) }
|
|
496
|
+
assert_equal [{ 'ok' => 1 }], handled
|
|
497
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
498
|
+
buried = producer.tubes.find('bury_fails').peek(:buried)
|
|
499
|
+
refute_nil buried, "the bad job was not buried"
|
|
500
|
+
assert_equal 'not json', buried.body
|
|
501
|
+
producer.close
|
|
502
|
+
ensure
|
|
503
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
504
|
+
end
|
|
505
|
+
|
|
506
|
+
it "raises NotConnected instead of looping, and the client resumes once the server is back" do
|
|
507
|
+
seen = []
|
|
508
|
+
@client.jobs.register('reconnect') do |job|
|
|
509
|
+
seen << job.body
|
|
510
|
+
@client.jobs.stop! if job.body == 'after restart'
|
|
511
|
+
end
|
|
512
|
+
@client.tubes.find('reconnect').put('before outage')
|
|
513
|
+
|
|
514
|
+
worker = Thread.new do
|
|
515
|
+
begin
|
|
516
|
+
@client.jobs.process!(:reserve_timeout => 1)
|
|
517
|
+
rescue StandardError => ex
|
|
518
|
+
ex
|
|
519
|
+
end
|
|
520
|
+
end
|
|
521
|
+
Timeout.timeout(5) { sleep 0.05 while seen.empty? }
|
|
522
|
+
stop_beanstalkd(@pid)
|
|
523
|
+
|
|
524
|
+
finished = worker.join(10)
|
|
525
|
+
worker.kill unless finished
|
|
526
|
+
assert finished, "process! was still running 10s after the server went away"
|
|
527
|
+
assert_kind_of Beaneater::NotConnected, worker.value
|
|
528
|
+
|
|
529
|
+
@pid = start_beanstalkd(@port)
|
|
530
|
+
producer = Beaneater.new("127.0.0.1:#{@port}")
|
|
531
|
+
producer.tubes.find('reconnect').put('after restart')
|
|
532
|
+
producer.close
|
|
533
|
+
|
|
534
|
+
Timeout.timeout(10) { @client.jobs.process!(:reserve_timeout => 1) }
|
|
535
|
+
assert_equal ['before outage', 'after restart'], seen
|
|
536
|
+
end
|
|
537
|
+
end
|
|
109
538
|
end # Beaneater::Jobs
|
data/test/stat_struct_test.rb
CHANGED
|
@@ -29,6 +29,11 @@ describe Beaneater::StatStruct do
|
|
|
29
29
|
assert_equal 'foo', @struct[:baz]
|
|
30
30
|
assert_equal 'demo', @struct[:"under_score"]
|
|
31
31
|
end
|
|
32
|
+
|
|
33
|
+
it "should look up hyphenated beanstalkd keys" do
|
|
34
|
+
assert_equal 'demo', @struct['under-score']
|
|
35
|
+
assert_equal 'demo', @struct[:"under-score"]
|
|
36
|
+
end
|
|
32
37
|
end # []
|
|
33
38
|
|
|
34
39
|
describe "for #keys" do
|
data/test/test_helper.rb
CHANGED
|
@@ -45,4 +45,49 @@ class MiniTest::Unit::TestCase
|
|
|
45
45
|
def teardown
|
|
46
46
|
flush_all
|
|
47
47
|
end
|
|
48
|
+
|
|
49
|
+
# The beanstalkd binary for tests that stop and restart their own server: BEANSTALKD_BIN, or
|
|
50
|
+
# `beanstalkd` on PATH. Skips the test when neither is available.
|
|
51
|
+
def beanstalkd_bin
|
|
52
|
+
bin = ENV['BEANSTALKD_BIN']
|
|
53
|
+
return bin if bin && !bin.empty?
|
|
54
|
+
on_path = ENV['PATH'].to_s.split(File::PATH_SEPARATOR).any? do |dir|
|
|
55
|
+
File.executable?(File.join(dir, 'beanstalkd'))
|
|
56
|
+
end
|
|
57
|
+
skip("beanstalkd not found: put it on PATH or set BEANSTALKD_BIN") unless on_path
|
|
58
|
+
'beanstalkd'
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# A free local port for a private beanstalkd
|
|
62
|
+
def free_port
|
|
63
|
+
server = TCPServer.new('127.0.0.1', 0)
|
|
64
|
+
server.addr[1]
|
|
65
|
+
ensure
|
|
66
|
+
server.close if server
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Starts a private beanstalkd on port and waits until it accepts connections.
|
|
70
|
+
#
|
|
71
|
+
# @return [Integer] the server's pid
|
|
72
|
+
def start_beanstalkd(port)
|
|
73
|
+
pid = Process.spawn(beanstalkd_bin, '-l', '127.0.0.1', '-p', port.to_s, [:out, :err] => File::NULL)
|
|
74
|
+
50.times do
|
|
75
|
+
begin
|
|
76
|
+
TCPSocket.new('127.0.0.1', port).close
|
|
77
|
+
return pid
|
|
78
|
+
rescue Errno::ECONNREFUSED
|
|
79
|
+
sleep 0.05
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
stop_beanstalkd(pid)
|
|
83
|
+
flunk "beanstalkd did not start on port #{port}"
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Kills a server started by start_beanstalkd
|
|
87
|
+
def stop_beanstalkd(pid)
|
|
88
|
+
Process.kill('KILL', pid)
|
|
89
|
+
Process.wait(pid)
|
|
90
|
+
rescue Errno::ESRCH, Errno::ECHILD
|
|
91
|
+
nil
|
|
92
|
+
end
|
|
48
93
|
end
|
data/test/tube_test.rb
CHANGED
|
@@ -96,11 +96,134 @@ describe Beaneater::Tube do
|
|
|
96
96
|
assert_equal 'hi: there! this: is a long message', @tube.peek(:ready).body['message']
|
|
97
97
|
end
|
|
98
98
|
|
|
99
|
+
it "should keep the connection usable when the custom parser raises" do
|
|
100
|
+
Beaneater.configure.job_parser = lambda { |b| JSON.parse(b) }
|
|
101
|
+
@tube.put('not json')
|
|
102
|
+
socket = @beanstalk.connection.connection
|
|
103
|
+
assert_raises(JSON::ParserError) { @tube.peek(:ready) }
|
|
104
|
+
assert_same socket, @beanstalk.connection.connection
|
|
105
|
+
assert_equal 'baz', @beanstalk.tubes.used.name
|
|
106
|
+
end
|
|
107
|
+
|
|
99
108
|
after do
|
|
100
109
|
Beaneater.configure.job_parser = lambda { |b| b }
|
|
101
110
|
end
|
|
102
111
|
end # peek
|
|
103
112
|
|
|
113
|
+
describe "for a job_parser that raises on reserve" do
|
|
114
|
+
before do
|
|
115
|
+
Beaneater.configure.job_parser = lambda { |b| JSON.parse(b) }
|
|
116
|
+
@tube = @beanstalk.tubes.find('poison_tube')
|
|
117
|
+
@poison_id = @tube.put('not json', :pri => 1)[:id]
|
|
118
|
+
@tube.put('{"ok":1}', :pri => 2)
|
|
119
|
+
@beanstalk.tubes.watch!('poison_tube')
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
after do
|
|
123
|
+
Beaneater.configure.job_parser = lambda { |b| b }
|
|
124
|
+
# The bad job is still reserved by this connection, which clearing the tube can't touch;
|
|
125
|
+
# left alone, it would be released when the socket is collected, into a later test.
|
|
126
|
+
begin
|
|
127
|
+
@beanstalk.connection.transmit("delete #{@poison_id}")
|
|
128
|
+
rescue Beaneater::NotFoundError
|
|
129
|
+
nil
|
|
130
|
+
end
|
|
131
|
+
cleanup_tubes!(['poison_tube'])
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
it "should raise the parser's own error from reserve, tagged with the job id" do
|
|
135
|
+
error = assert_raises(JSON::ParserError) { @beanstalk.tubes.reserve(0) }
|
|
136
|
+
assert_kind_of Beaneater::JobParseFailure, error
|
|
137
|
+
assert_equal @poison_id, error.job_id
|
|
138
|
+
assert_nil error.cause # the parser's error had none
|
|
139
|
+
Beaneater.configure.job_parser = lambda { |b| b }
|
|
140
|
+
assert_equal 'reserved', @beanstalk.connection.transmit("stats-job #{error.job_id}")[:body]['state']
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
it "should keep the parser's error intact: class, message, backtrace and cause" do
|
|
144
|
+
original = nil
|
|
145
|
+
Beaneater.configure.job_parser = lambda do |b|
|
|
146
|
+
begin
|
|
147
|
+
JSON.parse(b)
|
|
148
|
+
rescue JSON::ParserError => ex
|
|
149
|
+
original = ex
|
|
150
|
+
raise ArgumentError, 'not a job' # its cause is the ParserError
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
error = assert_raises(ArgumentError) { @beanstalk.tubes.reserve(0) }
|
|
154
|
+
assert_equal @poison_id, error.job_id
|
|
155
|
+
assert_equal 'not a job', error.message
|
|
156
|
+
assert_same original, error.cause
|
|
157
|
+
assert_operator error.backtrace.first, :include?, 'tube_test.rb'
|
|
158
|
+
refute_nil error.backtrace_locations
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# An exception class that can't be copied on any Ruby (clone calls initialize_copy)
|
|
162
|
+
class UncopyableError < StandardError
|
|
163
|
+
def initialize_copy(_original)
|
|
164
|
+
raise TypeError, "can't copy"
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# An exception class whose #cause is not an exception
|
|
169
|
+
class OddCauseError < StandardError
|
|
170
|
+
def cause
|
|
171
|
+
'not an exception'
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
it "should raise the parser's own exception, untagged, when it can't be copied" do
|
|
176
|
+
Beaneater.configure.job_parser = lambda { |b| raise UncopyableError, 'no copies' }
|
|
177
|
+
error = assert_raises(UncopyableError) { @beanstalk.tubes.reserve(0) }
|
|
178
|
+
assert_equal 'no copies', error.message
|
|
179
|
+
refute_kind_of Beaneater::JobParseFailure, error
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
it "should raise a frozen exception that can't be copied as it is" do
|
|
183
|
+
frozen = UncopyableError.new('frozen, no copies').freeze
|
|
184
|
+
Beaneater.configure.job_parser = lambda { |b| raise frozen }
|
|
185
|
+
error = assert_raises(UncopyableError) { @beanstalk.tubes.reserve(0) }
|
|
186
|
+
assert_same frozen, error
|
|
187
|
+
assert_nil error.cause
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
it "should tag the copy when the exception's cause is not an exception" do
|
|
191
|
+
Beaneater.configure.job_parser = lambda { |b| raise OddCauseError, 'odd' }
|
|
192
|
+
error = assert_raises(OddCauseError) { @beanstalk.tubes.reserve(0) }
|
|
193
|
+
assert_equal @poison_id, error.job_id
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
it "should not tag an exception the parser reuses, so a later error carries no old id" do
|
|
197
|
+
shared = JSON::ParserError.new('always this one')
|
|
198
|
+
Beaneater.configure.job_parser = lambda { |b| raise shared }
|
|
199
|
+
error = assert_raises(JSON::ParserError) { @beanstalk.tubes.reserve(0) }
|
|
200
|
+
assert_equal @poison_id, error.job_id
|
|
201
|
+
peeked = assert_raises(JSON::ParserError) { @tube.peek(:ready) } # not a reserve: not tagged
|
|
202
|
+
refute_kind_of Beaneater::JobParseFailure, peeked
|
|
203
|
+
refute_kind_of Beaneater::JobParseFailure, shared
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
it "should keep the connection, so the bad job stays reserved and the next job is delivered" do
|
|
207
|
+
socket = @beanstalk.connection.connection
|
|
208
|
+
assert_raises(JSON::ParserError) { @beanstalk.tubes.reserve(0) }
|
|
209
|
+
assert_same socket, @beanstalk.connection.connection
|
|
210
|
+
job = @beanstalk.tubes.reserve(0)
|
|
211
|
+
assert_equal({ 'ok' => 1 }, job.body)
|
|
212
|
+
job.delete
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
it "should keep the connection when the raising parser used the same client first" do
|
|
216
|
+
tube = 'poison_tube'
|
|
217
|
+
Beaneater.configure.job_parser = lambda { |b| @beanstalk.connection.transmit("use #{tube}"); JSON.parse(b) }
|
|
218
|
+
socket = @beanstalk.connection.connection
|
|
219
|
+
assert_raises(JSON::ParserError) { @beanstalk.tubes.reserve(0) }
|
|
220
|
+
assert_same socket, @beanstalk.connection.connection
|
|
221
|
+
job = @beanstalk.tubes.reserve(0)
|
|
222
|
+
assert_equal({ 'ok' => 1 }, job.body)
|
|
223
|
+
job.delete
|
|
224
|
+
end
|
|
225
|
+
end # job_parser that raises on reserve
|
|
226
|
+
|
|
104
227
|
describe "for #reserve" do
|
|
105
228
|
before do
|
|
106
229
|
@time = Time.now.to_i
|
data/test/tubes_test.rb
CHANGED
|
@@ -60,6 +60,13 @@ describe Beaneater::Tubes do
|
|
|
60
60
|
assert @beanstalk.tubes.all.map(&:name).include?(t)
|
|
61
61
|
end
|
|
62
62
|
end
|
|
63
|
+
|
|
64
|
+
it 'should return tube names that look like numbers or dates as strings' do
|
|
65
|
+
names = ['1.10', '2026-10-30', 'true', 'a-b']
|
|
66
|
+
names.each { |t| @beanstalk.tubes.find(t).put 'x' }
|
|
67
|
+
all = @beanstalk.tubes.all.map(&:name)
|
|
68
|
+
names.each { |t| assert_includes all, t }
|
|
69
|
+
end
|
|
63
70
|
end # all
|
|
64
71
|
|
|
65
72
|
describe "for Enumerable" do
|
|
@@ -157,8 +164,16 @@ describe Beaneater::Tubes do
|
|
|
157
164
|
@tube.reserve.delete
|
|
158
165
|
@tube.put "foo reserve #{@time}", :ttr => 1
|
|
159
166
|
@beanstalk.tubes.watch 'tube'
|
|
160
|
-
@beanstalk.tubes.reserve
|
|
167
|
+
job = @beanstalk.tubes.reserve
|
|
161
168
|
assert_raises(Beaneater::DeadlineSoonError) { @beanstalk.tubes.reserve(0) }
|
|
169
|
+
ensure
|
|
170
|
+
# left reserved, the job would be released (TTR, or the socket being collected) into a
|
|
171
|
+
# later #reserve test
|
|
172
|
+
begin
|
|
173
|
+
job.delete if job
|
|
174
|
+
rescue Beaneater::NotFoundError
|
|
175
|
+
nil
|
|
176
|
+
end
|
|
162
177
|
end
|
|
163
178
|
|
|
164
179
|
end # reserve
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: beaneater
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Nico Taing
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-10-31 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: minitest
|
|
@@ -136,7 +136,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
136
136
|
requirements:
|
|
137
137
|
- - ">="
|
|
138
138
|
- !ruby/object:Gem::Version
|
|
139
|
-
version: '
|
|
139
|
+
version: '2.7'
|
|
140
140
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
141
141
|
requirements:
|
|
142
142
|
- - ">="
|