ione-rpc 1.0.0.pre0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,614 @@
1
+ # encoding: utf-8
2
+
3
+ require 'spec_helper'
4
+
5
+
6
+ module Ione
7
+ module Rpc
8
+ describe Client do
9
+ let :client do
10
+ ClientSpec::TestClient.new(codec, io_reactor: io_reactor, logger: logger, connection_timeout: 7, hosts: hosts)
11
+ end
12
+
13
+ let :hosts do
14
+ %w[node0.example.com:4321 node1.example.com:5432 node2.example.com:6543]
15
+ end
16
+
17
+ let :io_reactor do
18
+ running = [false]
19
+ r = double(:io_reactor)
20
+ r.stub(:running?) { running[0] }
21
+ r.stub(:start) do
22
+ running[0] = true
23
+ Future.resolved(r)
24
+ end
25
+ r.stub(:stop) do
26
+ running[0] = false
27
+ Future.resolved(r)
28
+ end
29
+ r.stub(:connect) do |host, port, _, &block|
30
+ Future.resolved(block.call(create_raw_connection(host, port)))
31
+ end
32
+ r
33
+ end
34
+
35
+ let :codec do
36
+ double(:codec)
37
+ end
38
+
39
+ let :logger do
40
+ double(:logger, warn: nil, info: nil, debug: nil)
41
+ end
42
+
43
+ def create_raw_connection(host, port)
44
+ connection = double("connection@#{host}:#{port}")
45
+ connection.stub(:host).and_return(host)
46
+ connection.stub(:port).and_return(port)
47
+ connection.stub(:on_data)
48
+ connection.stub(:on_closed)
49
+ connection.stub(:write)
50
+ connection.stub(:close)
51
+ connection
52
+ end
53
+
54
+ before do
55
+ codec.stub(:encode) { |input, channel| input }
56
+ end
57
+
58
+ describe '#start' do
59
+ it 'starts the reactor' do
60
+ client.start.value
61
+ io_reactor.should have_received(:start)
62
+ end
63
+
64
+ it 'returns a future that resolves to the client' do
65
+ client.start.value.should equal(client)
66
+ end
67
+
68
+ it 'connects to the specified hosts and ports using the specified connection timeout' do
69
+ client.start.value
70
+ io_reactor.should have_received(:connect).with('node0.example.com', 4321, 7)
71
+ io_reactor.should have_received(:connect).with('node1.example.com', 5432, 7)
72
+ io_reactor.should have_received(:connect).with('node2.example.com', 6543, 7)
73
+ end
74
+
75
+ it 'accepts the hosts and ports as an array of pairs' do
76
+ hosts = [['node0.example.com', 4321], ['node1.example.com', '5432']]
77
+ client = ClientSpec::TestClient.new(codec, hosts: hosts, io_reactor: io_reactor, logger: logger, connection_timeout: 7)
78
+ client.start.value
79
+ io_reactor.should have_received(:connect).with('node0.example.com', 4321, 7)
80
+ io_reactor.should have_received(:connect).with('node1.example.com', 5432, 7)
81
+ end
82
+
83
+ it 'creates a protocol handler for each connection' do
84
+ client.start.value
85
+ client.created_connections.map(&:host).should == %w[node0.example.com node1.example.com node2.example.com]
86
+ client.created_connections.map(&:port).should == [4321, 5432, 6543]
87
+ end
88
+
89
+ it 'logs when the connection succeeds' do
90
+ client.start.value
91
+ logger.should have_received(:info).with(/connected to node0.example\.com:4321/i)
92
+ logger.should have_received(:info).with(/connected to node1.example\.com:5432/i)
93
+ logger.should have_received(:info).with(/connected to node2.example\.com:6543/i)
94
+ end
95
+
96
+ it 'attempts to connect again when a connection fails' do
97
+ connection_attempts = 0
98
+ attempts_by_host = Hash.new(0)
99
+ io_reactor.stub(:schedule_timer).and_return(Future.resolved)
100
+ io_reactor.stub(:connect) do |host, port, _, &block|
101
+ if host == 'node1.example.com'
102
+ Future.resolved(block.call(create_raw_connection(host, port)))
103
+ else
104
+ attempts_by_host[host] += 1
105
+ if attempts_by_host[host] < 10
106
+ Future.failed(StandardError.new('BORK'))
107
+ else
108
+ Future.resolved(block.call(create_raw_connection(host, port)))
109
+ end
110
+ end
111
+ end
112
+ client.start.value
113
+ io_reactor.should have_received(:connect).with('node1.example.com', anything, anything).once
114
+ attempts_by_host['node0.example.com'].should == 10
115
+ attempts_by_host['node2.example.com'].should == 10
116
+ end
117
+
118
+ it 'doubles the time it waits between connection attempts up to 10x the connection timeout' do
119
+ connection_attempts = 0
120
+ timeouts = []
121
+ io_reactor.stub(:schedule_timer) do |n|
122
+ timeouts << n
123
+ Future.resolved
124
+ end
125
+ io_reactor.stub(:connect) do |host, port, _, &block|
126
+ if host == 'node1.example.com'
127
+ connection_attempts += 1
128
+ if connection_attempts < 10
129
+ Future.failed(StandardError.new('BORK'))
130
+ else
131
+ Future.resolved(block.call(create_raw_connection(host, port)))
132
+ end
133
+ else
134
+ Future.resolved(block.call(create_raw_connection(host, port)))
135
+ end
136
+ end
137
+ client.start.value
138
+ timeouts.should == [7, 14, 28, 56, 70, 70, 70, 70, 70]
139
+ end
140
+
141
+ it 'stops trying to reconnect when the reactor is stopped' do
142
+ io_reactor.stub(:schedule_timer) do
143
+ promise = Promise.new
144
+ Thread.start do
145
+ sleep(0.01)
146
+ promise.fulfill
147
+ end
148
+ promise.future
149
+ end
150
+ io_reactor.stub(:connect).and_return(Future.failed(StandardError.new('BORK')))
151
+ f = client.start
152
+ io_reactor.stop.value
153
+ expect { f.value }.to raise_error(Io::ConnectionError, /IO reactor stopped while connecting/i)
154
+ end
155
+
156
+ it 'logs each connection attempt and failure' do
157
+ connection_attempts = 0
158
+ io_reactor.stub(:schedule_timer).and_return(Future.resolved)
159
+ io_reactor.stub(:connect) do |host, port, _, &block|
160
+ if host == 'node1.example.com'
161
+ connection_attempts += 1
162
+ if connection_attempts < 3
163
+ Future.failed(StandardError.new('BORK'))
164
+ else
165
+ Future.resolved(block.call(create_raw_connection(host, port)))
166
+ end
167
+ else
168
+ Future.resolved(block.call(create_raw_connection(host, port)))
169
+ end
170
+ end
171
+ client.start.value
172
+ logger.should have_received(:debug).with(/connecting to node0\.example\.com:4321/i).once
173
+ logger.should have_received(:debug).with(/connecting to node1\.example\.com:5432/i).exactly(3).times
174
+ logger.should have_received(:debug).with(/connecting to node2\.example\.com:6543/i).once
175
+ logger.should have_received(:warn).with(/failed connecting to node1\.example\.com:5432, will try again in \d+s/i).exactly(2).times
176
+ end
177
+
178
+ it 'sends a startup message once the connection has been established' do
179
+ client.start.value
180
+ client.created_connections.each { |c| c.requests.first.should == 'STARTUP' }
181
+ end
182
+ end
183
+
184
+ describe '#stop' do
185
+ it 'stops the reactor' do
186
+ client.stop.value
187
+ io_reactor.should have_received(:stop)
188
+ end
189
+
190
+ it 'returns a future that resolves to the client' do
191
+ client.stop.value.should equal(client)
192
+ end
193
+ end
194
+
195
+ describe '#send_request' do
196
+ context 'when expecting a response' do
197
+ before do
198
+ client.start.value
199
+ client.created_connections.each do |connection|
200
+ connection.stub(:send_message).with('PING').and_return(Future.resolved('PONG'))
201
+ end
202
+ end
203
+
204
+ it 'returns a future that resolves to the response from the server' do
205
+ client.send_request('PING').value.should == 'PONG'
206
+ end
207
+
208
+ it 'returns a failed future when called when not connected' do
209
+ client.stop.value
210
+ expect { client.send_request('PING').value }.to raise_error(Io::ConnectionError)
211
+ end
212
+ end
213
+
214
+ it 'uses the codec to encode frames' do
215
+ client.start.value
216
+ client.send_request('PING').value
217
+ codec.should have_received(:encode).with('PING', anything)
218
+ end
219
+
220
+ it 'chooses the connection to receive the request randomly' do
221
+ client.start.value
222
+ 1000.times { client.send_request('PING') }
223
+ client.created_connections.each do |connection|
224
+ connection.requests.size.should be_within(50).of(333)
225
+ end
226
+ end
227
+
228
+ context 'when the client chooses the connection per request' do
229
+ it 'asks the client to choose a connection to send the request on' do
230
+ client.override_choose_connection do |connections, request|
231
+ if request == 'PING'
232
+ connections[0]
233
+ elsif request == 'PONG'
234
+ connections[1]
235
+ else
236
+ connections[2]
237
+ end
238
+ end
239
+ client.start.value
240
+ client.send_request('PING').value
241
+ client.send_request('PING').value
242
+ client.send_request('PONG').value
243
+ startup_requests = 1
244
+ client.created_connections.map { |c| c.requests.size - startup_requests }.sort.should == [0, 1, 2]
245
+ end
246
+
247
+ it 'fails the request when #choose_connection raises an error' do
248
+ client.override_choose_connection do |connections, request|
249
+ raise 'Bork'
250
+ end
251
+ client.start.value
252
+ f = client.send_request('PING')
253
+ expect { f.value }.to raise_error('Bork')
254
+ end
255
+
256
+ it 'fails the request when #choose_connection returns nil' do
257
+ client.override_choose_connection do |connections, request|
258
+ nil
259
+ end
260
+ client.start.value
261
+ f = client.send_request('PING')
262
+ expect { f.value }.to raise_error(Io::ConnectionError)
263
+ end
264
+ end
265
+ end
266
+
267
+ describe '#connected?' do
268
+ it 'returns false before the client is started' do
269
+ client.should_not be_connected
270
+ end
271
+
272
+ it 'returns true when the client has started' do
273
+ client.start.value
274
+ client.should be_connected
275
+ end
276
+
277
+ it 'returns false when the client has been stopped' do
278
+ client.start.value
279
+ client.stop.value
280
+ client.should_not be_connected
281
+ end
282
+
283
+ it 'returns false when the connection has closed' do
284
+ client.start.value
285
+ io_reactor.stub(:schedule_timer).and_return(Future.resolved)
286
+ io_reactor.stub(:connect).and_return(Future.failed(StandardError.new('BORK')))
287
+ client.created_connections.each { |connection| connection.closed_listener.call }
288
+ client.should_not be_connected
289
+ end
290
+ end
291
+
292
+ describe '#add_host' do
293
+ context 'when called before the client is started' do
294
+ it 'connects to the host when the client starts' do
295
+ client.add_host('new.example.com', 3333)
296
+ io_reactor.should_not have_received(:connect)
297
+ client.start.value
298
+ io_reactor.should have_received(:connect).with('new.example.com', 3333, 7)
299
+ end
300
+
301
+ it 'returns a future that resolves to the client when the host has been connected to' do
302
+ f = client.add_host('new.example.com', 3333)
303
+ f.should_not be_resolved
304
+ client.start.value
305
+ f.value.should equal(client)
306
+ end
307
+ end
308
+
309
+ context 'when called after the client has started' do
310
+ it 'connects to the host immediately' do
311
+ client.start.value
312
+ io_reactor.should_not have_received(:connect).with('new.example.com', 3333, anything)
313
+ client.add_host('new.example.com', 3333)
314
+ io_reactor.should have_received(:connect).with('new.example.com', 3333, 7)
315
+ end
316
+
317
+ it 'returns a future that resolves to the client when the host has been connected to' do
318
+ client.start.value
319
+ f = client.add_host('new.example.com', 3333)
320
+ f.value.should equal(client)
321
+ end
322
+ end
323
+
324
+ it 'accepts a single host:port string' do
325
+ client.add_host('new.example.com:3333')
326
+ io_reactor.should_not have_received(:connect)
327
+ client.start.value
328
+ io_reactor.should have_received(:connect).with('new.example.com', 3333, 7)
329
+ end
330
+
331
+ it 'does not connect again if the host was already known' do
332
+ client.add_host('new.example.com:3333')
333
+ client.add_host('new.example.com:3333')
334
+ client.start.value
335
+ io_reactor.should have_received(:connect).with('new.example.com', 3333, 7).once
336
+ client.add_host('new.example.com:3333')
337
+ io_reactor.should have_received(:connect).with('new.example.com', 3333, 7).once
338
+ end
339
+ end
340
+
341
+ describe '#remove_host' do
342
+ it 'returns a future that resolves to the client future' do
343
+ client.remove_host('new.example.com:3333').value.should equal(client)
344
+ end
345
+
346
+ it 'does not connect to the host when the client starts' do
347
+ client.remove_host('node0.example.com', 4321)
348
+ client.start.value
349
+ io_reactor.should_not have_received(:connect).with('node0.example.com', 4321, anything)
350
+ end
351
+
352
+ it 'disconnects from the host when client has started' do
353
+ client.start.value
354
+ client.remove_host('node0.example.com', 4321)
355
+ io_reactor.should have_received(:connect).with('node0.example.com', 4321, anything)
356
+ connection = client.created_connections.find { |c| c.host == 'node0.example.com' }
357
+ connection.should be_closed
358
+ end
359
+
360
+ context 'when the connection had already closed, but not reconnected' do
361
+ let :timer_promise do
362
+ Promise.new
363
+ end
364
+
365
+ before do
366
+ client.start.value
367
+ io_reactor.stub(:schedule_timer).and_return(timer_promise.future)
368
+ io_reactor.stub(:connect).with('node0.example.com', 4321, anything).and_return(Future.failed(StandardError.new('BORK')))
369
+ connection = client.created_connections.find { |c| c.host == 'node0.example.com' }
370
+ connection.closed_listener.call(StandardError.new('BORK'))
371
+ client.remove_host('node0.example.com', 4321)
372
+ timer_promise.fulfill
373
+ end
374
+
375
+ it 'stops the reconnection attempts' do
376
+ io_reactor.should have_received(:connect).with('node0.example.com', 4321, anything).exactly(3).times
377
+ end
378
+
379
+ it 'logs that it stopped attempting to reconnect' do
380
+ logger.should have_received(:info).with('Not reconnecting to node0.example.com:4321')
381
+ end
382
+ end
383
+
384
+ context 'when the connection is being established' do
385
+ it 'closes the connection' do
386
+ connection_promise = Promise.new
387
+ connection_creation_block = nil
388
+ io_reactor.stub(:connect).with('node0.example.com', 4321, anything) do |_, _, _, &block|
389
+ connection_creation_block = block
390
+ connection_promise.future
391
+ end
392
+ client.start
393
+ client.remove_host('node0.example.com', 4321)
394
+ connection = create_raw_connection('node0.example.com', 4321)
395
+ connection_promise.fulfill(connection_creation_block.call(connection))
396
+ connection.should have_received(:close)
397
+ end
398
+ end
399
+ end
400
+
401
+ context 'when disconnected' do
402
+ it 'logs that the connection closed' do
403
+ client.start.value
404
+ client.created_connections.find { |c| c.host == 'node1.example.com' }.closed_listener.call
405
+ logger.should have_received(:info).with(/connection to node1\.example\.com:5432 closed/i)
406
+ logger.should_not have_received(:info).with(/node0\.example\.com closed/i)
407
+ end
408
+
409
+ it 'logs that the connection closed unexpectedly' do
410
+ client.start.value
411
+ client.created_connections.find { |c| c.host == 'node1.example.com' }.closed_listener.call(StandardError.new('BORK'))
412
+ logger.should have_received(:warn).with(/connection to node1\.example\.com:5432 closed unexpectedly: BORK/i)
413
+ logger.should_not have_received(:warn).with(/node0\.example\.com/i)
414
+ end
415
+
416
+ it 'logs when requests fail' do
417
+ client.start.value
418
+ client.created_connections.each { |connection| connection.stub(:send_message).with('PING').and_return(Future.failed(StandardError.new('BORK'))) }
419
+ client.send_request('PING')
420
+ logger.should have_received(:warn).with(/request failed: BORK/i)
421
+ end
422
+
423
+ it 'attempts to reconnect' do
424
+ client.start.value
425
+ client.created_connections.find { |c| c.host == 'node1.example.com' }.closed_listener.call(StandardError.new('BORK'))
426
+ io_reactor.should have_received(:connect).exactly(4).times
427
+ end
428
+
429
+ it 'does not attempt to reconnect on a clean close' do
430
+ client.start.value
431
+ client.created_connections.find { |c| c.host == 'node1.example.com' }.closed_listener.call
432
+ io_reactor.should have_received(:connect).exactly(3).times
433
+ end
434
+
435
+ it 'does not attempt to reconnect when #reconnect? returns false' do
436
+ client.reconnect = 2
437
+ client.start.value
438
+ io_reactor.stub(:schedule_timer).and_return(Future.resolved)
439
+ io_reactor.stub(:connect).and_return(Future.failed(StandardError.new('BORK')))
440
+ client.created_connections.find { |c| c.host == 'node1.example.com' }.closed_listener.call(StandardError.new('BURK'))
441
+ io_reactor.should have_received(:connect).with('node1.example.com', 5432, anything).exactly(3).times
442
+ end
443
+
444
+ it 'allows a manual reconnect after stopping automatic reconnections' do
445
+ client.reconnect = 1
446
+ client.start.value
447
+ io_reactor.stub(:schedule_timer).and_return(Future.resolved)
448
+ io_reactor.stub(:connect).and_return(Future.failed(StandardError.new('BORK')))
449
+ client.created_connections.find { |c| c.host == 'node1.example.com' }.closed_listener.call(StandardError.new('BURK'))
450
+ io_reactor.should have_received(:connect).with('node1.example.com', 5432, anything).exactly(2).times
451
+ io_reactor.stub(:connect) { |h, p, _, &block| Future.resolved(block.call(create_raw_connection(h, p))) }
452
+ client.add_host('node1.example.com', 5432).value
453
+ io_reactor.should have_received(:connect).with('node1.example.com', 5432, anything).exactly(3).times
454
+ end
455
+
456
+ it 'runs the same connection logic as #connect' do
457
+ connection_attempts = 0
458
+ connection_attempts_by_host = Hash.new(0)
459
+ io_reactor.stub(:schedule_timer).and_return(Future.resolved)
460
+ io_reactor.stub(:connect) do |host, port, _, &block|
461
+ connection_attempts_by_host[host] += 1
462
+ if host == 'node1.example.com'
463
+ connection_attempts += 1
464
+ if connection_attempts > 1 && connection_attempts < 10
465
+ Future.failed(StandardError.new('BORK'))
466
+ else
467
+ Future.resolved(block.call(create_raw_connection(host, port)))
468
+ end
469
+ else
470
+ Future.resolved(block.call(create_raw_connection(host, port)))
471
+ end
472
+ end
473
+ client.start.value
474
+ client.created_connections.find { |c| c.host == 'node1.example.com' }.closed_listener.call(StandardError.new('BORK'))
475
+ connection_attempts_by_host['node0.example.com'].should == 1
476
+ connection_attempts_by_host['node1.example.com'].should == 10
477
+ connection_attempts_by_host['node2.example.com'].should == 1
478
+ end
479
+ end
480
+
481
+ context 'with multiple connections' do
482
+ before do
483
+ client.start.value
484
+ end
485
+
486
+ it 'sends requests over a random connection' do
487
+ 1000.times do
488
+ client.send_request('PING')
489
+ end
490
+ request_fractions = client.created_connections.each_with_object({}) { |connection, acc| acc[connection.host] = connection.requests.size/1000.0 }
491
+ request_fractions['node0.example.com'].should be_within(0.1).of(0.33)
492
+ request_fractions['node1.example.com'].should be_within(0.1).of(0.33)
493
+ request_fractions['node2.example.com'].should be_within(0.1).of(0.33)
494
+ end
495
+
496
+ it 'retries the request when it failes because a connection closed' do
497
+ promises = [Promise.new, Promise.new]
498
+ counter = 0
499
+ received_requests = []
500
+ client.created_connections.each do |connection|
501
+ connection.stub(:send_message) do |request|
502
+ received_requests << request
503
+ promises[counter].future.tap { counter += 1 }
504
+ end
505
+ end
506
+ client.send_request('PING')
507
+ sleep 0.01 until counter > 0
508
+ promises[0].fail(Io::ConnectionClosedError.new('CLOSED BORK'))
509
+ promises[1].fulfill('PONG')
510
+ received_requests.should have(2).items
511
+ end
512
+
513
+ it 'logs when a request is retried' do
514
+ client.created_connections.each do |connection|
515
+ connection.stub(:send_message) do
516
+ connection.closed_listener.call
517
+ Future.failed(Io::ConnectionClosedError.new('CLOSED BORK'))
518
+ end
519
+ end
520
+ client.send_request('PING')
521
+ logger.should have_received(:warn).with(/request failed because the connection closed, retrying/i).at_least(1).times
522
+ end
523
+ end
524
+ end
525
+ end
526
+ end
527
+
528
+ module ClientSpec
529
+ class TestClient < Ione::Rpc::Client
530
+ attr_reader :created_connections
531
+
532
+ def initialize(*)
533
+ super
534
+ @created_connections = []
535
+ end
536
+
537
+ def create_connection(raw_connection)
538
+ peer_connection = super
539
+ @created_connections << TestConnection.new(raw_connection, peer_connection)
540
+ @created_connections.last
541
+ end
542
+
543
+ def initialize_connection(connection)
544
+ super.flat_map do
545
+ send_request('STARTUP', connection)
546
+ end
547
+ end
548
+
549
+ def override_choose_connection(&chooser)
550
+ @connection_chooser = chooser
551
+ end
552
+
553
+ def choose_connection(connections, request)
554
+ if @connection_chooser
555
+ @connection_chooser.call(connections, request)
556
+ else
557
+ super
558
+ end
559
+ end
560
+
561
+ def reconnect=(state)
562
+ @reconnect = state
563
+ end
564
+
565
+ def reconnect?(host, port, attempts)
566
+ if defined?(@reconnect)
567
+ if @reconnect.is_a?(Integer)
568
+ @reconnect > attempts
569
+ else
570
+ @reconnect
571
+ end
572
+ else
573
+ super
574
+ end
575
+ end
576
+ end
577
+
578
+ class TestConnection
579
+ attr_reader :closed_listener, :requests
580
+
581
+ def initialize(raw_connection, peer_connection)
582
+ @raw_connection = raw_connection
583
+ @peer_connection = peer_connection
584
+ @requests = []
585
+ end
586
+
587
+ def closed?
588
+ !!@closed
589
+ end
590
+
591
+ def close
592
+ @closed = true
593
+ @peer_connection.close
594
+ end
595
+
596
+ def on_closed(&listener)
597
+ @closed_listener = listener
598
+ end
599
+
600
+ def host
601
+ @raw_connection.host
602
+ end
603
+
604
+ def port
605
+ @raw_connection.port
606
+ end
607
+
608
+ def send_message(request)
609
+ @requests << request
610
+ @peer_connection.send_message(request)
611
+ Ione::Future.resolved
612
+ end
613
+ end
614
+ end