beaneater 1.1.3 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +44 -0
- data/CHANGELOG.md +23 -0
- data/Gemfile +2 -0
- data/README.md +29 -1
- data/Rakefile +2 -0
- data/beaneater.gemspec +2 -0
- data/examples/demo.rb +2 -0
- data/lib/beaneater/configuration.rb +10 -0
- data/lib/beaneater/connection.rb +315 -57
- data/lib/beaneater/errors.rb +16 -0
- data/lib/beaneater/job/collection.rb +44 -6
- data/lib/beaneater/job/record.rb +2 -0
- data/lib/beaneater/job.rb +2 -0
- data/lib/beaneater/stats/fast_struct.rb +2 -0
- data/lib/beaneater/stats/stat_struct.rb +8 -1
- data/lib/beaneater/stats.rb +2 -0
- data/lib/beaneater/tube/collection.rb +2 -0
- data/lib/beaneater/tube/record.rb +2 -0
- data/lib/beaneater/tube.rb +2 -0
- data/lib/beaneater/version.rb +3 -1
- data/lib/beaneater.rb +2 -0
- data/test/beaneater_test.rb +2 -0
- data/test/connection_test.rb +612 -12
- data/test/errors_test.rb +2 -0
- data/test/job_test.rb +14 -0
- data/test/jobs_test.rb +431 -0
- data/test/prompt_regexp_test.rb +2 -0
- data/test/stat_struct_test.rb +7 -0
- data/test/stats_test.rb +2 -0
- data/test/test_helper.rb +47 -0
- data/test/tube_test.rb +125 -0
- data/test/tubes_test.rb +18 -1
- metadata +5 -4
data/test/connection_test.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
# test/connection_test.rb
|
|
2
4
|
|
|
3
5
|
require File.expand_path('../test_helper', __FILE__)
|
|
@@ -39,6 +41,152 @@ describe Beaneater::Connection do
|
|
|
39
41
|
end
|
|
40
42
|
end # new
|
|
41
43
|
|
|
44
|
+
describe 'for timeout configuration' do
|
|
45
|
+
after do
|
|
46
|
+
Beaneater.configure do |config|
|
|
47
|
+
config.connect_timeout = nil
|
|
48
|
+
config.resolv_timeout = nil
|
|
49
|
+
config.read_timeout = nil
|
|
50
|
+
config.write_timeout = nil
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
it "should pass connect_timeout and resolv_timeout to TCPSocket.new on Ruby >= 3.0" do
|
|
55
|
+
skip("connect_timeout/resolv_timeout kwargs require Ruby >= 3.0") if RUBY_VERSION < "3.0"
|
|
56
|
+
|
|
57
|
+
Beaneater.configure do |config|
|
|
58
|
+
config.connect_timeout = 2
|
|
59
|
+
config.resolv_timeout = 2
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
real_socket = TCPSocket.new('localhost', 11300)
|
|
63
|
+
TCPSocket.expects(:new).with('localhost', 11300, connect_timeout: 2, resolv_timeout: 2).returns(real_socket)
|
|
64
|
+
|
|
65
|
+
bc = Beaneater::Connection.new('localhost')
|
|
66
|
+
assert_kind_of TCPSocket, bc.connection
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
it "should ignore connect_timeout and resolv_timeout on Ruby < 3.0" do
|
|
70
|
+
skip("connect_timeout/resolv_timeout kwargs are supported on Ruby >= 3.0") if RUBY_VERSION >= "3.0"
|
|
71
|
+
|
|
72
|
+
Beaneater.configure do |config|
|
|
73
|
+
config.connect_timeout = 2
|
|
74
|
+
config.resolv_timeout = 4
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
real_socket = TCPSocket.new('localhost', 11300)
|
|
78
|
+
TCPSocket.expects(:new).with('localhost', 11300).returns(real_socket)
|
|
79
|
+
|
|
80
|
+
bc = Beaneater::Connection.new('localhost')
|
|
81
|
+
assert_same real_socket, bc.connection
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# A server that accepts connections and then neither reads nor replies.
|
|
85
|
+
def with_silent_server
|
|
86
|
+
server = TCPServer.new('127.0.0.1', 0)
|
|
87
|
+
clients = []
|
|
88
|
+
acceptor = Thread.new { loop { clients << server.accept } }
|
|
89
|
+
yield "127.0.0.1:#{server.addr[1]}", clients
|
|
90
|
+
ensure
|
|
91
|
+
acceptor.kill if acceptor
|
|
92
|
+
clients.each(&:close)
|
|
93
|
+
server.close if server
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def elapsed
|
|
97
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
98
|
+
yield
|
|
99
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
it "should raise SocketTimeout and drop the connection when no reply arrives within read_timeout" do
|
|
103
|
+
Beaneater.configure { |config| config.read_timeout = 0.5 }
|
|
104
|
+
with_silent_server do |address, clients|
|
|
105
|
+
bc = Beaneater::Connection.new(address)
|
|
106
|
+
time = elapsed do
|
|
107
|
+
assert_raises(Beaneater::SocketTimeout) { Timeout.timeout(5) { bc.transmit('stats') } }
|
|
108
|
+
end
|
|
109
|
+
assert_in_delta 0.5, time, 0.4
|
|
110
|
+
assert_nil bc.connection
|
|
111
|
+
# the next command opens a new connection rather than reading the late reply
|
|
112
|
+
assert_raises(Beaneater::SocketTimeout) { Timeout.timeout(5) { bc.transmit('stats') } }
|
|
113
|
+
assert_equal 2, clients.size
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
it "should raise SocketTimeout when a command cannot be written within write_timeout" do
|
|
118
|
+
Beaneater.configure { |config| config.write_timeout = 0.5 }
|
|
119
|
+
with_silent_server do |address|
|
|
120
|
+
bc = Beaneater::Connection.new(address)
|
|
121
|
+
body = 'x' * (64 * 1024 * 1024) # more than the socket buffers hold
|
|
122
|
+
time = elapsed do
|
|
123
|
+
assert_raises(Beaneater::SocketTimeout) do
|
|
124
|
+
Timeout.timeout(5) { bc.transmit("put 0 0 100 #{body.bytesize}\r\n#{body}") }
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
assert_operator time, :<, 2
|
|
128
|
+
assert_nil bc.connection
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
it "should accept timeouts given as numeric strings" do
|
|
133
|
+
Beaneater.configure do |config|
|
|
134
|
+
config.read_timeout = '0.5'
|
|
135
|
+
config.write_timeout = '0.5'
|
|
136
|
+
end
|
|
137
|
+
with_silent_server do |address|
|
|
138
|
+
bc = Beaneater::Connection.new(address)
|
|
139
|
+
assert_raises(Beaneater::SocketTimeout) { Timeout.timeout(5) { bc.transmit('stats') } }
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
it "should read a reply that is already there when the deadline has passed" do
|
|
144
|
+
Beaneater.configure { |config| config.read_timeout = 0 }
|
|
145
|
+
server = TCPServer.new('127.0.0.1', 0)
|
|
146
|
+
# answers before the command arrives, so the reply is waiting when the client polls
|
|
147
|
+
responder = Thread.new { peer = server.accept; peer.write("USING eager\r\n"); peer }
|
|
148
|
+
bc = Beaneater::Connection.new("127.0.0.1:#{server.addr[1]}")
|
|
149
|
+
peer = responder.value
|
|
150
|
+
sleep 0.2
|
|
151
|
+
res = bc.transmit('use eager')
|
|
152
|
+
assert_equal 'USING', res[:status]
|
|
153
|
+
assert_equal 'eager', res[:id]
|
|
154
|
+
ensure
|
|
155
|
+
peer.close if peer
|
|
156
|
+
server.close if server
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
it "should wait reserve-with-timeout's own timeout on top of read_timeout" do
|
|
160
|
+
Beaneater.configure { |config| config.read_timeout = 0.5 }
|
|
161
|
+
client = Beaneater.new('localhost')
|
|
162
|
+
client.tubes.watch!('read_timeout_empty')
|
|
163
|
+
time = elapsed do
|
|
164
|
+
assert_raises(Beaneater::TimedOutError) { client.tubes.reserve(1) }
|
|
165
|
+
end
|
|
166
|
+
assert_operator time, :>=, 0.9
|
|
167
|
+
assert client.connection.connection, "connection should stay open"
|
|
168
|
+
client.close
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
it "should not cut off a plain reserve with read_timeout" do
|
|
172
|
+
Beaneater.configure { |config| config.read_timeout = 0.5 }
|
|
173
|
+
client = Beaneater.new('localhost')
|
|
174
|
+
client.tubes.watch!('read_timeout_reserve')
|
|
175
|
+
producer = Thread.new do
|
|
176
|
+
sleep 1
|
|
177
|
+
Beaneater.new('localhost').tubes['read_timeout_reserve'].put('late')
|
|
178
|
+
end
|
|
179
|
+
job = nil
|
|
180
|
+
time = elapsed { job = Timeout.timeout(5) { client.tubes.reserve } }
|
|
181
|
+
assert_equal 'late', job.body
|
|
182
|
+
assert_operator time, :>=, 0.9
|
|
183
|
+
job.delete
|
|
184
|
+
producer.join
|
|
185
|
+
client.close
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
end # timeout configuration
|
|
189
|
+
|
|
42
190
|
describe 'for #transmit' do
|
|
43
191
|
before do
|
|
44
192
|
@host = 'localhost'
|
|
@@ -51,6 +199,24 @@ describe Beaneater::Connection do
|
|
|
51
199
|
assert_equal 'OK', res[:status]
|
|
52
200
|
end
|
|
53
201
|
|
|
202
|
+
it "should keep the types of stats values without mangling them" do
|
|
203
|
+
@bc.transmit "use stats-tube.1"
|
|
204
|
+
id = @bc.transmit("put 0 0 100 1\r\nX")[:id]
|
|
205
|
+
tube = @bc.transmit('stats-tube stats-tube.1')[:body]
|
|
206
|
+
assert_equal 'stats-tube.1', tube['name']
|
|
207
|
+
assert_equal 1, tube['current-jobs-ready']
|
|
208
|
+
|
|
209
|
+
stats = @bc.transmit('stats')[:body]
|
|
210
|
+
assert_kind_of Integer, stats['current-connections']
|
|
211
|
+
assert_kind_of Float, stats['rusage-utime']
|
|
212
|
+
assert_kind_of String, stats['version']
|
|
213
|
+
assert_kind_of String, stats['hostname']
|
|
214
|
+
assert_equal false, stats['draining']
|
|
215
|
+
ensure
|
|
216
|
+
@bc.transmit "delete #{id}" if id
|
|
217
|
+
@bc.transmit "use default"
|
|
218
|
+
end
|
|
219
|
+
|
|
54
220
|
it "should return id" do
|
|
55
221
|
res = @bc.transmit "put 0 0 100 1\r\nX"
|
|
56
222
|
assert res[:id]
|
|
@@ -74,16 +240,16 @@ describe Beaneater::Connection do
|
|
|
74
240
|
end
|
|
75
241
|
|
|
76
242
|
it "should retry command with success after one connection failure" do
|
|
77
|
-
TCPSocket.any_instance.expects(:
|
|
243
|
+
TCPSocket.any_instance.expects(:readpartial).times(2).
|
|
78
244
|
raises(EOFError.new).then.
|
|
79
|
-
returns("DELETED 56\
|
|
245
|
+
returns("DELETED 56\r\n")
|
|
80
246
|
|
|
81
247
|
res = @bc.transmit "delete 56\r\n"
|
|
82
248
|
assert_equal 'DELETED', res[:status]
|
|
83
249
|
end
|
|
84
250
|
|
|
85
251
|
it "should fail after exceeding retries with DrainingError" do
|
|
86
|
-
TCPSocket.any_instance.expects(:
|
|
252
|
+
TCPSocket.any_instance.expects(:readpartial).times(3).
|
|
87
253
|
raises(Beaneater::UnexpectedResponse.from_status("DRAINING", "delete 56"))
|
|
88
254
|
|
|
89
255
|
assert_raises(Beaneater::DrainingError) { @bc.transmit "delete 56\r\n" }
|
|
@@ -92,7 +258,7 @@ describe Beaneater::Connection do
|
|
|
92
258
|
it "should fail after exceeding reconnect max retries" do
|
|
93
259
|
# next connection attempts should fail
|
|
94
260
|
TCPSocket.stubs(:new).times(3).raises(Errno::ECONNREFUSED.new)
|
|
95
|
-
TCPSocket.any_instance.stubs(:
|
|
261
|
+
TCPSocket.any_instance.stubs(:readpartial).times(1).raises(EOFError.new)
|
|
96
262
|
|
|
97
263
|
assert_raises(Beaneater::NotConnected) { @bc.transmit "delete 56\r\n" }
|
|
98
264
|
end
|
|
@@ -100,19 +266,280 @@ describe Beaneater::Connection do
|
|
|
100
266
|
it "tubes_watched are restored after reconnect" do
|
|
101
267
|
client = Beaneater.new('127.0.0.1:11300')
|
|
102
268
|
client.tubes.watch! "another"
|
|
269
|
+
socket = client.connection.connection
|
|
270
|
+
socket.expects(:readpartial).once.raises(EOFError.new) # this socket only
|
|
271
|
+
|
|
272
|
+
assert_equal %w[another], client.tubes.watched.map(&:name)
|
|
273
|
+
refute_same socket, client.connection.connection
|
|
274
|
+
ensure
|
|
275
|
+
client.close if client
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
it "should raise NotConnected when reconnecting fails with any connect error" do
|
|
279
|
+
TCPSocket.stubs(:new).raises(Errno::EHOSTUNREACH.new)
|
|
280
|
+
TCPSocket.any_instance.stubs(:readpartial).raises(EOFError.new)
|
|
281
|
+
|
|
282
|
+
assert_raises(Beaneater::NotConnected) { @bc.transmit "stats", retry_interval: 0 }
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
it "should reconnect on the next command after draining retries ran out" do
|
|
286
|
+
TCPSocket.any_instance.stubs(:readpartial).raises(Beaneater::UnexpectedResponse.from_status("DRAINING", "stats"))
|
|
287
|
+
assert_raises(Beaneater::DrainingError) { @bc.transmit "stats", retry_interval: 0 }
|
|
103
288
|
|
|
104
|
-
TCPSocket.
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
289
|
+
TCPSocket.any_instance.unstub(:readpartial)
|
|
290
|
+
assert_equal 'OK', @bc.transmit('stats')[:status]
|
|
291
|
+
end
|
|
292
|
+
|
|
293
|
+
it "should drop the connection when a command is interrupted, so the next command reads its own reply" do
|
|
294
|
+
client = Beaneater.new('localhost')
|
|
295
|
+
client.tubes.watch!('interrupted')
|
|
296
|
+
assert_raises(Timeout::Error) { Timeout.timeout(0.3) { client.tubes.reserve } }
|
|
297
|
+
|
|
298
|
+
producer = Beaneater.new('localhost')
|
|
299
|
+
producer.tubes.find('interrupted').put('late job')
|
|
300
|
+
producer.close
|
|
301
|
+
sleep 0.1 # let a reserve still pending on the old socket take the job
|
|
302
|
+
|
|
303
|
+
assert_equal %w[interrupted], client.tubes.watched.map(&:name)
|
|
304
|
+
job = client.tubes.reserve(1)
|
|
305
|
+
assert_equal 'late job', job.body
|
|
306
|
+
job.delete
|
|
307
|
+
ensure
|
|
308
|
+
client.close if client
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
it "should keep the connection after an error reply" do
|
|
312
|
+
socket = @bc.connection
|
|
313
|
+
assert_raises(Beaneater::NotFoundError) { @bc.transmit 'peek 4294967295' }
|
|
314
|
+
assert_same socket, @bc.connection
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
it "restores the watched tubes again when the connection drops while restoring them" do
|
|
318
|
+
# A minimal beanstalkd that drops the 1st connection on 'stats' and the 2nd on 'ignore'
|
|
319
|
+
server = TCPServer.new('127.0.0.1', 0)
|
|
320
|
+
accepted = 0
|
|
321
|
+
fake = Thread.new do
|
|
322
|
+
loop do
|
|
323
|
+
sock = server.accept
|
|
324
|
+
accepted += 1
|
|
325
|
+
Thread.new(sock, accepted) do |peer, n|
|
|
326
|
+
watched = ['default']
|
|
327
|
+
while (line = peer.gets("\r\n"))
|
|
328
|
+
cmd, arg = line.chomp("\r\n").split(' ', 2)
|
|
329
|
+
if (n == 1 && cmd == 'stats') || (n == 2 && cmd == 'ignore')
|
|
330
|
+
peer.close
|
|
331
|
+
break
|
|
332
|
+
end
|
|
333
|
+
reply = case cmd
|
|
334
|
+
when 'watch' then watched |= [arg]; "WATCHING #{watched.size}\r\n"
|
|
335
|
+
when 'ignore'
|
|
336
|
+
if watched == [arg] then "NOT_IGNORED\r\n"
|
|
337
|
+
else watched -= [arg]; "WATCHING #{watched.size}\r\n"
|
|
338
|
+
end
|
|
339
|
+
when 'list-tubes-watched', 'stats'
|
|
340
|
+
body = cmd == 'stats' ? "---\nuptime: 1\n" : "---\n" + watched.map { |t| "- #{t}\n" }.join
|
|
341
|
+
"OK #{body.bytesize}\r\n#{body}\r\n"
|
|
342
|
+
else "UNKNOWN_COMMAND\r\n"
|
|
343
|
+
end
|
|
344
|
+
peer.write(reply)
|
|
345
|
+
end
|
|
109
346
|
end
|
|
347
|
+
end
|
|
348
|
+
end
|
|
349
|
+
client = Beaneater.new("127.0.0.1:#{server.addr[1]}")
|
|
350
|
+
client.tubes.watch!('restore-a', 'restore-b')
|
|
351
|
+
|
|
352
|
+
assert_equal({ 'uptime' => 1 }, Timeout.timeout(5) { client.connection.transmit('stats', retry_interval: 0)[:body] })
|
|
353
|
+
assert_equal %w[restore-a restore-b], client.tubes.watched.map(&:name)
|
|
354
|
+
assert_equal 3, accepted
|
|
355
|
+
ensure
|
|
356
|
+
client.close if client
|
|
357
|
+
fake.kill if fake
|
|
358
|
+
server.close if server
|
|
359
|
+
end
|
|
110
360
|
|
|
111
|
-
|
|
361
|
+
it "gives up when every new connection is dropped as soon as a command is sent" do
|
|
362
|
+
# like a proxy that accepts connections while its backend is down
|
|
363
|
+
server = TCPServer.new('127.0.0.1', 0)
|
|
364
|
+
accepted = 0
|
|
365
|
+
acceptor = Thread.new do
|
|
366
|
+
loop do
|
|
367
|
+
peer = server.accept
|
|
368
|
+
accepted += 1
|
|
369
|
+
if accepted == 1
|
|
370
|
+
Thread.new(peer) { |p| p.gets("\r\n"); p.write("USING x\r\n"); p.gets("\r\n"); p.close }
|
|
371
|
+
else
|
|
372
|
+
Thread.new(peer) { |p| p.gets("\r\n"); p.close }
|
|
373
|
+
end
|
|
112
374
|
end
|
|
113
|
-
|
|
375
|
+
end
|
|
376
|
+
bc = Beaneater::Connection.new("127.0.0.1:#{server.addr[1]}")
|
|
377
|
+
assert_equal 'USING', bc.transmit('use x')[:status]
|
|
378
|
+
assert_raises(Beaneater::NotConnected) { Timeout.timeout(5) { bc.transmit('stats', retry_interval: 0) } }
|
|
379
|
+
assert_operator accepted, :<=, Beaneater::Connection::MAX_RETRIES + 1
|
|
380
|
+
ensure
|
|
381
|
+
acceptor.kill if acceptor
|
|
382
|
+
server.close if server
|
|
383
|
+
end
|
|
114
384
|
|
|
115
|
-
|
|
385
|
+
describe "after the server was down for longer than the reconnect retries" do
|
|
386
|
+
before do
|
|
387
|
+
@port = free_port
|
|
388
|
+
@pid = start_beanstalkd(@port)
|
|
389
|
+
@client = Beaneater.new("127.0.0.1:#{@port}")
|
|
390
|
+
end
|
|
391
|
+
|
|
392
|
+
after do
|
|
393
|
+
@client.close if @client
|
|
394
|
+
stop_beanstalkd(@pid) if @pid
|
|
395
|
+
end
|
|
396
|
+
|
|
397
|
+
it "reconnects on the next command and restores the used and watched tubes" do
|
|
398
|
+
@client.tubes.watch('reconnect-watched')
|
|
399
|
+
@client.tubes.use('reconnect-used')
|
|
400
|
+
|
|
401
|
+
stop_beanstalkd(@pid)
|
|
402
|
+
assert_raises(Beaneater::NotConnected) do
|
|
403
|
+
@client.connection.transmit('list-tubes', retry_interval: 0.01)
|
|
404
|
+
end
|
|
405
|
+
|
|
406
|
+
@pid = start_beanstalkd(@port)
|
|
407
|
+
assert_equal 'reconnect-used', @client.tubes.used.name
|
|
408
|
+
assert_equal %w[default reconnect-watched], @client.tubes.watched.map(&:name).sort
|
|
409
|
+
end
|
|
410
|
+
|
|
411
|
+
it "lets threads sharing the client wait out their reconnect retries side by side" do
|
|
412
|
+
stop_beanstalkd(@pid)
|
|
413
|
+
now = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
|
|
414
|
+
started = now.call
|
|
415
|
+
waits = 4.times.map do
|
|
416
|
+
Thread.new do
|
|
417
|
+
t = now.call
|
|
418
|
+
begin
|
|
419
|
+
@client.connection.transmit('stats', retry_interval: 0.5)
|
|
420
|
+
rescue Beaneater::NotConnected
|
|
421
|
+
end
|
|
422
|
+
now.call - t
|
|
423
|
+
end
|
|
424
|
+
end.map(&:value)
|
|
425
|
+
# each thread waits out about 1 s of retries; one after another that would be 4 s
|
|
426
|
+
assert_operator now.call - started, :<, 2, "waits: #{waits.inspect}"
|
|
427
|
+
|
|
428
|
+
@pid = start_beanstalkd(@port)
|
|
429
|
+
assert_equal 'OK', @client.connection.transmit('stats')[:status]
|
|
430
|
+
end
|
|
431
|
+
|
|
432
|
+
it "lets threads sharing the client recover when the server restarts under them" do
|
|
433
|
+
@client.tubes.use('restart-used')
|
|
434
|
+
stop = false
|
|
435
|
+
workers = 2.times.map do |i|
|
|
436
|
+
Thread.new do
|
|
437
|
+
until stop
|
|
438
|
+
begin
|
|
439
|
+
i.zero? ? @client.tubes['restart-used'].put('x') : @client.tubes.used
|
|
440
|
+
rescue Beaneater::NotConnected, IOError
|
|
441
|
+
sleep 0.01
|
|
442
|
+
end
|
|
443
|
+
end
|
|
444
|
+
end
|
|
445
|
+
end
|
|
446
|
+
5.times do
|
|
447
|
+
sleep 0.3
|
|
448
|
+
stop_beanstalkd(@pid)
|
|
449
|
+
@pid = start_beanstalkd(@port)
|
|
450
|
+
end
|
|
451
|
+
sleep 0.3
|
|
452
|
+
stop = true
|
|
453
|
+
stuck = workers.reject { |t| t.join(5) }
|
|
454
|
+
stuck.each(&:kill)
|
|
455
|
+
assert_empty stuck, "threads still blocked 5s after the last restart"
|
|
456
|
+
assert_equal 'restart-used', Timeout.timeout(5) { @client.tubes.used.name }
|
|
457
|
+
end
|
|
458
|
+
|
|
459
|
+
it "does not reopen a connection closed while it was reconnecting" do
|
|
460
|
+
stop_beanstalkd(@pid)
|
|
461
|
+
result = nil
|
|
462
|
+
worker = Thread.new do
|
|
463
|
+
begin
|
|
464
|
+
@client.connection.transmit('stats', retry_interval: 0.5)
|
|
465
|
+
result = :answered
|
|
466
|
+
rescue Exception => ex
|
|
467
|
+
result = ex.class
|
|
468
|
+
end
|
|
469
|
+
end
|
|
470
|
+
sleep 0.2 # the worker is between reconnect attempts
|
|
471
|
+
@client.close
|
|
472
|
+
@pid = start_beanstalkd(@port)
|
|
473
|
+
assert worker.join(5), "worker still running"
|
|
474
|
+
assert_equal Beaneater::NotConnected, result
|
|
475
|
+
assert_nil @client.connection.connection
|
|
476
|
+
end
|
|
477
|
+
end
|
|
478
|
+
|
|
479
|
+
describe "while the server is draining" do
|
|
480
|
+
before do
|
|
481
|
+
@port = free_port
|
|
482
|
+
@pid = start_beanstalkd(@port)
|
|
483
|
+
@client = Beaneater.new("127.0.0.1:#{@port}")
|
|
484
|
+
end
|
|
485
|
+
|
|
486
|
+
after do
|
|
487
|
+
@client.close if @client
|
|
488
|
+
stop_beanstalkd(@pid) if @pid
|
|
489
|
+
end
|
|
490
|
+
|
|
491
|
+
it "lets threads sharing the client wait out their draining retries side by side" do
|
|
492
|
+
Process.kill('USR1', @pid) # beanstalkd answers DRAINING to put from now on
|
|
493
|
+
sleep 0.2
|
|
494
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
495
|
+
errors = 4.times.map do
|
|
496
|
+
Thread.new do
|
|
497
|
+
begin
|
|
498
|
+
@client.connection.transmit("put 0 0 100 1\r\nx", retry_interval: 0.3)
|
|
499
|
+
rescue Beaneater::DrainingError => ex
|
|
500
|
+
ex.class
|
|
501
|
+
end
|
|
502
|
+
end
|
|
503
|
+
end.map(&:value)
|
|
504
|
+
assert_equal [Beaneater::DrainingError] * 4, errors
|
|
505
|
+
# each put waits 2 x 0.3 s between its tries; one after another that would be 2.4 s
|
|
506
|
+
assert_operator Process.clock_gettime(Process::CLOCK_MONOTONIC) - started, :<, 1.5
|
|
507
|
+
end
|
|
508
|
+
|
|
509
|
+
it "lets Timeout stop a command pausing between retries while another thread holds the client" do
|
|
510
|
+
@client.tubes.find('slow-parse').put('slow')
|
|
511
|
+
@client.tubes.watch!('slow-parse')
|
|
512
|
+
Process.kill('USR1', @pid) # draining: put is refused, reserve still works
|
|
513
|
+
sleep 0.2
|
|
514
|
+
Beaneater.configure.job_parser = lambda { |body| sleep 2 if body == 'slow'; body }
|
|
515
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
516
|
+
put = Thread.new do
|
|
517
|
+
begin
|
|
518
|
+
Timeout.timeout(0.5) { @client.connection.transmit("put 0 0 100 1\r\nx", retry_interval: 0.3) }
|
|
519
|
+
rescue Exception => ex
|
|
520
|
+
[ex.class, Process.clock_gettime(Process::CLOCK_MONOTONIC) - started]
|
|
521
|
+
end
|
|
522
|
+
end
|
|
523
|
+
sleep 0.1 # the put got DRAINING and is pausing before its next try
|
|
524
|
+
holder = Thread.new { @client.connection.transmit('reserve-with-timeout 0') } # holds the client ~2 s
|
|
525
|
+
error, at = put.value
|
|
526
|
+
holder.join
|
|
527
|
+
assert_equal Timeout::Error, error
|
|
528
|
+
assert_operator at, :<, 1, "Timeout fired after #{at.round(2)}s"
|
|
529
|
+
ensure
|
|
530
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
531
|
+
end
|
|
532
|
+
|
|
533
|
+
it "uses the default retry interval between DRAINING retries when given nil" do
|
|
534
|
+
Process.kill('USR1', @pid)
|
|
535
|
+
sleep 0.2
|
|
536
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
537
|
+
assert_raises(Beaneater::DrainingError) do
|
|
538
|
+
Timeout.timeout(5) { @client.connection.transmit("put 0 0 100 1\r\nx", retry_interval: nil) }
|
|
539
|
+
end
|
|
540
|
+
elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
541
|
+
assert_in_delta 2 * Beaneater::Connection::DEFAULT_RETRY_INTERVAL, elapsed, 0.5
|
|
542
|
+
end
|
|
116
543
|
end
|
|
117
544
|
end # transmit
|
|
118
545
|
|
|
@@ -128,5 +555,178 @@ describe Beaneater::Connection do
|
|
|
128
555
|
assert_nil @bc.connection
|
|
129
556
|
assert_raises(Beaneater::NotConnected) { @bc.transmit 'stats' }
|
|
130
557
|
end
|
|
558
|
+
|
|
559
|
+
# Removes the job a test left in tube name. Its client was closed with the job reserved, so
|
|
560
|
+
# wait (briefly) for the server to release it before clearing the tube.
|
|
561
|
+
def clear_after_close(name)
|
|
562
|
+
cleaner = Beaneater.new('localhost')
|
|
563
|
+
20.times do
|
|
564
|
+
cleanup_tubes!([name], cleaner)
|
|
565
|
+
stats = begin
|
|
566
|
+
cleaner.tubes.find(name).stats
|
|
567
|
+
rescue Beaneater::NotFoundError
|
|
568
|
+
break # the tube is gone: no jobs left
|
|
569
|
+
end
|
|
570
|
+
break if stats.current_jobs_reserved.zero? && stats.current_jobs_ready.zero?
|
|
571
|
+
sleep 0.05
|
|
572
|
+
end
|
|
573
|
+
ensure
|
|
574
|
+
cleaner.close if cleaner
|
|
575
|
+
end
|
|
576
|
+
|
|
577
|
+
it "should close the socket once a command whose job_parser closed the client is over" do
|
|
578
|
+
client = Beaneater.new('localhost')
|
|
579
|
+
client.tubes.find('close-from-parser').put('x')
|
|
580
|
+
client.tubes.watch!('close-from-parser')
|
|
581
|
+
socket = client.connection.connection
|
|
582
|
+
Beaneater.configure.job_parser = lambda { |body| client.close; body }
|
|
583
|
+
assert_equal 'x', client.tubes.reserve(0).body
|
|
584
|
+
assert_nil client.connection.connection
|
|
585
|
+
assert socket.closed?, "the shut-down socket is still open"
|
|
586
|
+
ensure
|
|
587
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
588
|
+
clear_after_close('close-from-parser')
|
|
589
|
+
end
|
|
590
|
+
|
|
591
|
+
it "should not raise when the socket is closed by another thread while #close shuts it down" do
|
|
592
|
+
client = Beaneater.new('localhost')
|
|
593
|
+
client.tubes.find('close-race').put('x')
|
|
594
|
+
client.tubes.watch!('close-race')
|
|
595
|
+
client.connection.connection.stubs(:shutdown).raises(IOError, 'closed stream')
|
|
596
|
+
Beaneater.configure.job_parser = lambda { |body| client.close; body }
|
|
597
|
+
assert_equal 'x', client.tubes.reserve(0).body
|
|
598
|
+
ensure
|
|
599
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
600
|
+
clear_after_close('close-race')
|
|
601
|
+
end
|
|
602
|
+
|
|
603
|
+
describe "in a forked child" do
|
|
604
|
+
before do
|
|
605
|
+
skip("needs fork") unless Process.respond_to?(:fork)
|
|
606
|
+
@client = Beaneater.new('localhost')
|
|
607
|
+
cleanup_tubes!(['fork-close'], @client)
|
|
608
|
+
@client.tubes.find('fork-close').put('work')
|
|
609
|
+
@client.tubes.watch!('fork-close')
|
|
610
|
+
end
|
|
611
|
+
|
|
612
|
+
after do
|
|
613
|
+
Beaneater.configure.job_parser = lambda { |body| body }
|
|
614
|
+
@client.close if @client
|
|
615
|
+
cleaner = Beaneater.new('localhost')
|
|
616
|
+
cleanup_tubes!(['fork-close'], cleaner) # a failed run leaves the job behind
|
|
617
|
+
cleaner.close
|
|
618
|
+
end
|
|
619
|
+
|
|
620
|
+
# The parent still holds the job it reserved: another client cannot get it, and the
|
|
621
|
+
# parent can delete it on its own connection.
|
|
622
|
+
def assert_parent_keeps(job)
|
|
623
|
+
other = Beaneater.new('localhost')
|
|
624
|
+
other.tubes.watch!('fork-close')
|
|
625
|
+
assert_raises(Beaneater::TimedOutError) { other.tubes.reserve(0) }
|
|
626
|
+
assert_equal 'DELETED', @client.connection.transmit("delete #{job.id}")[:status]
|
|
627
|
+
ensure
|
|
628
|
+
other.close if other
|
|
629
|
+
end
|
|
630
|
+
|
|
631
|
+
it "should leave the parent's connection alone" do
|
|
632
|
+
job = @client.tubes.reserve(0)
|
|
633
|
+
child = fork { @client.close; exit!(0) }
|
|
634
|
+
Process.wait(child)
|
|
635
|
+
assert_parent_keeps(job)
|
|
636
|
+
end
|
|
637
|
+
|
|
638
|
+
it "should leave the parent's connection alone when forked during a command" do
|
|
639
|
+
child = nil
|
|
640
|
+
Beaneater.configure.job_parser = lambda { |body| child = fork { @client.close; exit!(0) }; body }
|
|
641
|
+
job = @client.tubes.reserve(0)
|
|
642
|
+
Beaneater.configure.job_parser = lambda { |body| body } # no second fork if anything below fails
|
|
643
|
+
Process.wait(child)
|
|
644
|
+
assert_parent_keeps(job)
|
|
645
|
+
end
|
|
646
|
+
end
|
|
647
|
+
|
|
648
|
+
it "should end a reserve when a signal handler on the same thread closes the client" do
|
|
649
|
+
# Guards the contract: the reserve ends with NotConnected. It also runs the case that
|
|
650
|
+
# crashed Ruby 3.3 (closing the socket under the wait), but that crash came in about one
|
|
651
|
+
# run in four, so 20 runs catch it with high probability, not certainty. A child process
|
|
652
|
+
# keeps a crash from taking the suite down.
|
|
653
|
+
script = <<-RUBY
|
|
654
|
+
require 'beaneater'
|
|
655
|
+
20.times do
|
|
656
|
+
client = Beaneater.new('localhost')
|
|
657
|
+
client.tubes.watch!('close-from-trap')
|
|
658
|
+
Signal.trap('USR2') { client.close }
|
|
659
|
+
Thread.new { sleep 0.05; Process.kill('USR2', Process.pid) }
|
|
660
|
+
puts(begin; client.tubes.reserve; 'reserved'; rescue Exception => ex; ex.class.name; end)
|
|
661
|
+
end
|
|
662
|
+
RUBY
|
|
663
|
+
lib = File.expand_path('../../lib', __FILE__)
|
|
664
|
+
out = IO.popen([RbConfig.ruby, '-I', lib, '-e', script]) do |child|
|
|
665
|
+
begin
|
|
666
|
+
Timeout.timeout(60) { child.read }
|
|
667
|
+
rescue Timeout::Error
|
|
668
|
+
Process.kill('KILL', child.pid)
|
|
669
|
+
flunk "the child was still running after 60s"
|
|
670
|
+
end
|
|
671
|
+
end
|
|
672
|
+
assert $?.success?, "child failed (#{$?.inspect}) after: #{out.lines.size} reserves"
|
|
673
|
+
assert_equal ['Beaneater::NotConnected'] * 20, out.lines.map(&:chomp)
|
|
674
|
+
end
|
|
675
|
+
|
|
676
|
+
it "should end a reserve in another thread when a signal handler closes the client" do
|
|
677
|
+
# Same contract and same caveat as the test above (the 3.3 crash is caught only with high
|
|
678
|
+
# probability), with the reserve in a second thread and the main thread idle.
|
|
679
|
+
script = <<-RUBY
|
|
680
|
+
require 'beaneater'
|
|
681
|
+
20.times do
|
|
682
|
+
client = Beaneater.new('localhost')
|
|
683
|
+
client.tubes.watch!('close-from-trap-other-thread')
|
|
684
|
+
trap_error = '-'
|
|
685
|
+
Signal.trap('USR2') do
|
|
686
|
+
begin
|
|
687
|
+
client.close
|
|
688
|
+
rescue Exception => ex
|
|
689
|
+
trap_error = ex.class.name
|
|
690
|
+
end
|
|
691
|
+
end
|
|
692
|
+
waiter = Thread.new { begin; client.tubes.reserve; 'reserved'; rescue Exception => ex; ex.class.name; end }
|
|
693
|
+
sleep 0.05 # the waiter is blocked in reserve
|
|
694
|
+
Process.kill('USR2', Process.pid)
|
|
695
|
+
puts "\#{waiter.join(5) ? waiter.value : 'still blocked'} \#{trap_error}"
|
|
696
|
+
end
|
|
697
|
+
RUBY
|
|
698
|
+
lib = File.expand_path('../../lib', __FILE__)
|
|
699
|
+
out = IO.popen([RbConfig.ruby, '-I', lib, '-e', script]) do |child|
|
|
700
|
+
begin
|
|
701
|
+
Timeout.timeout(60) { child.read }
|
|
702
|
+
rescue Timeout::Error
|
|
703
|
+
Process.kill('KILL', child.pid)
|
|
704
|
+
flunk "the child was still running after 60s"
|
|
705
|
+
end
|
|
706
|
+
end
|
|
707
|
+
assert $?.success?, "child failed (#{$?.inspect}) after: #{out.lines.size} reserves"
|
|
708
|
+
assert_equal ['Beaneater::NotConnected -'] * 20, out.lines.map(&:chomp)
|
|
709
|
+
end
|
|
710
|
+
|
|
711
|
+
it "should unblock a reserve waiting in another thread" do
|
|
712
|
+
client = Beaneater.new('localhost')
|
|
713
|
+
client.tubes.watch!('close-unblocks-reserve')
|
|
714
|
+
result = nil
|
|
715
|
+
waiter = Thread.new do
|
|
716
|
+
begin
|
|
717
|
+
client.tubes.reserve
|
|
718
|
+
result = :reserved
|
|
719
|
+
rescue Exception => ex
|
|
720
|
+
result = ex
|
|
721
|
+
end
|
|
722
|
+
end
|
|
723
|
+
sleep 0.3 # the waiter is blocked in reserve
|
|
724
|
+
client.close
|
|
725
|
+
assert waiter.join(1), "reserve still blocked 1s after close"
|
|
726
|
+
assert_kind_of Beaneater::NotConnected, result
|
|
727
|
+
assert_nil client.connection.connection
|
|
728
|
+
ensure
|
|
729
|
+
waiter.kill if waiter
|
|
730
|
+
end
|
|
131
731
|
end # close
|
|
132
732
|
end # Beaneater::Connection
|