bunny 0.9.0.pre10 → 0.9.0.pre11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +7 -0
  2. data/.ruby-version +1 -1
  3. data/.travis.yml +4 -7
  4. data/ChangeLog.md +79 -0
  5. data/Gemfile +3 -1
  6. data/README.md +1 -1
  7. data/benchmarks/basic_publish/with_128K_messages.rb +35 -0
  8. data/benchmarks/basic_publish/with_1k_messages.rb +35 -0
  9. data/benchmarks/basic_publish/with_4K_messages.rb +35 -0
  10. data/benchmarks/basic_publish/with_64K_messages.rb +35 -0
  11. data/benchmarks/channel_open.rb +28 -0
  12. data/benchmarks/queue_declare.rb +29 -0
  13. data/benchmarks/queue_declare_and_bind.rb +29 -0
  14. data/benchmarks/queue_declare_bind_and_delete.rb +29 -0
  15. data/benchmarks/write_vs_write_nonblock.rb +49 -0
  16. data/bunny.gemspec +3 -3
  17. data/lib/bunny.rb +2 -0
  18. data/lib/bunny/channel.rb +31 -35
  19. data/lib/bunny/concurrent/continuation_queue.rb +10 -0
  20. data/lib/bunny/concurrent/linked_continuation_queue.rb +4 -2
  21. data/lib/bunny/exceptions.rb +5 -2
  22. data/lib/bunny/heartbeat_sender.rb +6 -4
  23. data/lib/bunny/queue.rb +3 -0
  24. data/lib/bunny/{main_loop.rb → reader_loop.rb} +5 -8
  25. data/lib/bunny/session.rb +93 -48
  26. data/lib/bunny/socket.rb +37 -3
  27. data/lib/bunny/test_kit.rb +26 -0
  28. data/lib/bunny/transport.rb +39 -33
  29. data/lib/bunny/version.rb +1 -1
  30. data/profiling/basic_publish/with_4K_messages.rb +33 -0
  31. data/spec/higher_level_api/integration/consistent_hash_exchange_spec.rb +10 -11
  32. data/spec/higher_level_api/integration/queue_declare_spec.rb +55 -13
  33. data/spec/issues/issue100_spec.rb +29 -27
  34. data/spec/issues/issue78_spec.rb +54 -52
  35. data/spec/stress/channel_open_stress_with_single_threaded_connection_spec.rb +0 -22
  36. data/spec/stress/concurrent_consumers_stress_spec.rb +2 -1
  37. data/spec/stress/concurrent_publishers_stress_spec.rb +7 -10
  38. data/spec/stress/long_running_consumer_spec.rb +83 -0
  39. data/spec/unit/concurrent/condition_spec.rb +7 -5
  40. data/spec/unit/concurrent/linked_continuation_queue_spec.rb +35 -0
  41. metadata +48 -44
@@ -20,8 +20,15 @@ module Bunny
20
20
  end
21
21
  end
22
22
 
23
+ # Reads given number of bytes with an optional timeout
24
+ #
25
+ # @param [Integer] count How many bytes to read
26
+ # @param [Integer] timeout Timeout
27
+ #
28
+ # @return [String] Data read from the socket
29
+ # @api public
23
30
  def read_fully(count, timeout = nil)
24
- return nil if @eof
31
+ return nil if @__bunny_socket_eof_flag__
25
32
 
26
33
  value = ''
27
34
  begin
@@ -30,8 +37,9 @@ module Bunny
30
37
  break if value.bytesize >= count
31
38
  end
32
39
  rescue EOFError
33
- @eof = true
34
- rescue Errno::EAGAIN, Errno::EWOULDBLOCK
40
+ # @eof will break Rubinius' TCPSocket implementation. MK.
41
+ @__bunny_socket_eof_flag__ = true
42
+ rescue Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitReadable
35
43
  if IO.select([self], nil, nil, timeout)
36
44
  retry
37
45
  else
@@ -39,6 +47,32 @@ module Bunny
39
47
  end
40
48
  end
41
49
  value
50
+ end # read_fully
51
+
52
+ # Writes provided data using IO#write_nonblock, taking care of handling
53
+ # of exceptions it raises when writing would fail (e.g. due to socket buffer
54
+ # being full).
55
+ #
56
+ # IMPORTANT: this method will mutate (slice) the argument. Pass in duplicates
57
+ # if this is not appropriate in your case.
58
+ #
59
+ # @param [String] data Data to write
60
+ #
61
+ # @api public
62
+ def write_nonblock_fully(data)
63
+ return nil if @__bunny_socket_eof_flag__
64
+
65
+ begin
66
+ while !data.empty?
67
+ written = self.write_nonblock(data)
68
+ data.slice!(0, written)
69
+ end
70
+ rescue Errno::EWOULDBLOCK, Errno::EAGAIN
71
+ IO.select([], [self])
72
+ retry
73
+ end
74
+
75
+ data.bytesize
42
76
  end
43
77
  end
44
78
  end
@@ -0,0 +1,26 @@
1
+ # -*- coding: utf-8 -*-
2
+ module Bunny
3
+ # Unit, integration and stress testing toolkit
4
+ class TestKit
5
+ class << self
6
+
7
+ # @return [Integer] Random integer in the range of [a, b]
8
+ # @api private
9
+ def random_in_range(a, b)
10
+ Range.new(a, b).to_a.sample
11
+ end
12
+
13
+ # @param [Integer] Lower bound of message size, in KB
14
+ # @param [Integer] Upper bound of message size, in KB
15
+ # @param [Integer] Random number to use in message generation
16
+ # @return [String] Message payload of length in the given range, with non-ASCII characters
17
+ def message_in_kb(a, b, i)
18
+ s = "Ю#{i}"
19
+ n = random_in_range(a, b) / s.bytesize
20
+
21
+ s * n * 1024
22
+ end
23
+
24
+ end
25
+ end
26
+ end
@@ -45,6 +45,8 @@ module Bunny
45
45
  @connect_timeout = nil if @connect_timeout == 0
46
46
  @disconnect_timeout = @read_write_timeout || @connect_timeout
47
47
 
48
+ @writes_mutex = Mutex.new
49
+
48
50
  initialize_socket
49
51
  connect
50
52
  end
@@ -84,13 +86,13 @@ module Bunny
84
86
  if @read_write_timeout
85
87
  Bunny::Timer.timeout(@read_write_timeout, Bunny::ClientTimeout) do
86
88
  if open?
87
- @socket.write(data)
89
+ @writes_mutex.synchronize { @socket.write(data) }
88
90
  @socket.flush
89
91
  end
90
92
  end
91
93
  else
92
94
  if open?
93
- @socket.write(data)
95
+ @writes_mutex.synchronize { @socket.write(data) }
94
96
  @socket.flush
95
97
  end
96
98
  end
@@ -104,12 +106,12 @@ module Bunny
104
106
  end
105
107
  end
106
108
  end
107
- alias send_raw write
108
109
 
109
110
  # Writes data to the socket without timeout checks
110
111
  def write_without_timeout(data)
111
112
  begin
112
- @socket.write(data)
113
+ @writes_mutex.synchronize { @socket.write(data) }
114
+ @socket.flush
113
115
  rescue SystemCallError, Bunny::ConnectionError, IOError => e
114
116
  close
115
117
 
@@ -121,6 +123,31 @@ module Bunny
121
123
  end
122
124
  end
123
125
 
126
+ # Sends frame to the peer.
127
+ #
128
+ # @raise [ConnectionClosedError]
129
+ # @private
130
+ def send_frame(frame)
131
+ if closed?
132
+ @session.handle_network_failure(ConnectionClosedError.new(frame))
133
+ else
134
+ write(frame.encode)
135
+ end
136
+ end
137
+
138
+ # Sends frame to the peer without timeout control.
139
+ #
140
+ # @raise [ConnectionClosedError]
141
+ # @private
142
+ def send_frame_without_timeout(frame)
143
+ if closed?
144
+ @session.handle_network_failure(ConnectionClosedError.new(frame))
145
+ else
146
+ write_without_timeout(frame.encode)
147
+ end
148
+ end
149
+
150
+
124
151
  def close(reason = nil)
125
152
  @socket.close unless @socket.closed?
126
153
  end
@@ -151,6 +178,14 @@ module Bunny
151
178
  # @private
152
179
  def read_next_frame(opts = {})
153
180
  header = @socket.read_fully(7)
181
+ # TODO: network issues here will sometimes cause
182
+ # the socket method return an empty string. We need to log
183
+ # and handle this better.
184
+ # type, channel, size = begin
185
+ # AMQ::Protocol::Frame.decode_header(header)
186
+ # rescue AMQ::Protocol::EmptyResponseError => e
187
+ # puts "Got AMQ::Protocol::EmptyResponseError, header is #{header.inspect}"
188
+ # end
154
189
  type, channel, size = AMQ::Protocol::Frame.decode_header(header)
155
190
  payload = @socket.read_fully(size)
156
191
  frame_end = @socket.read_fully(1)
@@ -166,35 +201,6 @@ module Bunny
166
201
  end
167
202
 
168
203
 
169
- # Sends frame to the peer.
170
- #
171
- # @raise [ConnectionClosedError]
172
- # @private
173
- def send_frame(frame)
174
- if closed?
175
- @session.handle_network_failure(ConnectionClosedError.new(frame))
176
- else
177
- frame.encode_to_array.each do |component|
178
- send_raw(component)
179
- end
180
- end
181
- end
182
-
183
- # Sends frame to the peer without timeout control.
184
- #
185
- # @raise [ConnectionClosedError]
186
- # @private
187
- def send_frame_without_timeout(frame)
188
- if closed?
189
- @session.handle_network_failure(ConnectionClosedError.new(frame))
190
- else
191
- frame.encode_to_array.each do |component|
192
- write_without_timeout(component)
193
- end
194
- end
195
- end
196
-
197
-
198
204
  def self.reacheable?(host, port, timeout)
199
205
  begin
200
206
  s = Bunny::Socket.open(host, port,
@@ -1,5 +1,5 @@
1
1
  # encoding: utf-8
2
2
 
3
3
  module Bunny
4
- VERSION = "0.9.0.pre10"
4
+ VERSION = "0.9.0.pre11"
5
5
  end
@@ -0,0 +1,33 @@
1
+ #!/usr/bin/env ruby
2
+ # encoding: utf-8
3
+
4
+ require "rubygems"
5
+ require "bunny"
6
+ require "ruby-prof"
7
+
8
+ conn = Bunny.new
9
+ conn.start
10
+
11
+ puts
12
+ puts "-" * 80
13
+ puts "Benchmarking on #{RUBY_DESCRIPTION}"
14
+
15
+ n = 50_000
16
+ ch = conn.create_channel
17
+ x = ch.default_exchange
18
+ s = "z" * 4096
19
+
20
+ # warm up the JIT, etc
21
+ puts "Doing a warmup run..."
22
+ 16000.times { x.publish(s, :routing_key => "anything") }
23
+
24
+ # give OS, the server and so on some time to catch
25
+ # up
26
+ sleep 2.0
27
+
28
+ result = RubyProf.profile do
29
+ n.times { x.publish(s, :routing_key => "anything") }
30
+ end
31
+
32
+ printer = RubyProf::FlatPrinter.new(result)
33
+ printer.print(STDOUT, {})
@@ -2,7 +2,7 @@
2
2
  require "spec_helper"
3
3
 
4
4
  unless ENV["CI"]
5
- describe "x-consistent-hash exchanges" do
5
+ describe "x-consistent-hash exchange" do
6
6
  let(:connection) do
7
7
  c = Bunny.new(:user => "bunny_gem", :password => "bunny_password", :vhost => "bunny_testbed")
8
8
  c.start
@@ -13,12 +13,11 @@ unless ENV["CI"]
13
13
  connection.close
14
14
  end
15
15
 
16
- let(:list) { Range.new(0, 30).to_a.map(&:to_s) }
16
+ let(:list) { Range.new(0, 6).to_a.map(&:to_s) }
17
17
 
18
- let(:n) { 20 }
19
- let(:m) { 10_000 }
18
+ let(:m) { 1500 }
20
19
 
21
- it "can be used" do
20
+ it "distributes messages between queues bound with the same routing key" do
22
21
  ch = connection.create_channel
23
22
  body = "сообщение"
24
23
  # requires the consistent hash exchange plugin,
@@ -29,8 +28,8 @@ unless ENV["CI"]
29
28
 
30
29
  qs = []
31
30
 
32
- q1 = ch.queue("", :exclusive => true).bind(x, :routing_key => "15")
33
- q2 = ch.queue("", :exclusive => true).bind(x, :routing_key => "15")
31
+ q1 = ch.queue("", :exclusive => true).bind(x, :routing_key => "5")
32
+ q2 = ch.queue("", :exclusive => true).bind(x, :routing_key => "5")
34
33
 
35
34
  sleep 1.0
36
35
 
@@ -38,12 +37,12 @@ unless ENV["CI"]
38
37
  m.times do
39
38
  x.publish(body, :routing_key => list.sample)
40
39
  end
41
- puts "Published #{(i + 1) * m} messages..."
40
+ puts "Published #{(i + 1) * m} tiny messages..."
42
41
  end
43
42
 
44
- sleep 4.0
45
- q1.message_count.should be > 1000
46
- q2.message_count.should be > 1000
43
+ sleep 2.0
44
+ q1.message_count.should be > 100
45
+ q2.message_count.should be > 100
47
46
 
48
47
  ch.close
49
48
  end
@@ -96,34 +96,76 @@ describe Bunny::Queue do
96
96
  end
97
97
 
98
98
 
99
- context "when queue is declared with additional arguments (e.g. message TTL)" do
99
+
100
+ context "when queue is declared with a different set of attributes" do
101
+ it "raises an exception" do
102
+ ch = connection.create_channel
103
+
104
+ q = ch.queue("bunny.tests.queues.auto-delete", :auto_delete => true, :durable => false)
105
+ expect {
106
+ # force re-declaration
107
+ ch.queue_declare("bunny.tests.queues.auto-delete", :auto_delete => false, :durable => true)
108
+ }.to raise_error(Bunny::PreconditionFailed)
109
+
110
+ ch.should be_closed
111
+ end
112
+ end
113
+
114
+
115
+ context "when queue is declared with message TTL" do
100
116
  let(:args) do
117
+ # in ms
101
118
  {"x-message-ttl" => 1000}
102
119
  end
103
120
 
104
- it "declares it with all the arguments provided" do
121
+ it "causes all messages in it to have a TTL" do
105
122
  ch = connection.create_channel
106
123
 
107
- q = ch.queue("bunny.tests.queues.with-arguments", :arguments => args)
124
+ q = ch.queue("bunny.tests.queues.with-arguments.ttl", :arguments => args, :exclusive => true)
108
125
  q.arguments.should == args
109
- q.delete
126
+
127
+ q.publish("xyzzy")
128
+ sleep 0.1
129
+
130
+ q.message_count.should == 1
131
+ sleep 1.5
132
+ q.message_count.should == 0
110
133
 
111
134
  ch.close
112
135
  end
113
136
  end
114
137
 
115
138
 
116
- context "when queue is declared with a different set of attributes" do
117
- it "raises an exception" do
118
- ch = connection.create_channel
139
+ unless ENV["CI"]
140
+ # requires RabbitMQ 3.1+
141
+ context "when queue is declared with bounded length" do
142
+ let(:n) { 10 }
143
+ let(:args) do
144
+ # in ms
145
+ {"x-max-length" => n}
146
+ end
119
147
 
120
- q = ch.queue("bunny.tests.queues.auto-delete", :auto_delete => true, :durable => false)
121
- expect {
122
- # force re-declaration
123
- ch.queue_declare("bunny.tests.queues.auto-delete", :auto_delete => false, :durable => true)
124
- }.to raise_error(Bunny::PreconditionFailed)
148
+ # see http://www.rabbitmq.com/maxlength.html for more info
149
+ it "causes the queue to be bounded" do
150
+ ch = connection.create_channel
125
151
 
126
- ch.should be_closed
152
+ q = ch.queue("bunny.tests.queues.with-arguments.max-length", :arguments => args, :exclusive => true)
153
+ q.arguments.should == args
154
+
155
+ (n * 10).times do
156
+ q.publish("xyzzy")
157
+ end
158
+
159
+ q.message_count.should == n
160
+ (n * 5).times do
161
+ q.publish("xyzzy")
162
+ end
163
+
164
+ q.message_count.should == n
165
+ q.delete
166
+
167
+ ch.close
168
+ end
127
169
  end
128
170
  end
129
171
  end
@@ -1,40 +1,42 @@
1
1
  require "spec_helper"
2
2
 
3
- describe Bunny::Channel, "#basic_publish" do
4
- let(:connection) do
5
- c = Bunny.new(:user => "bunny_gem",
6
- :password => "bunny_password",
7
- :vhost => "bunny_testbed",
8
- :socket_timeout => 0)
9
- c.start
10
- c
11
- end
3
+ unless ENV["CI"]
4
+ describe Bunny::Channel, "#basic_publish" do
5
+ let(:connection) do
6
+ c = Bunny.new(:user => "bunny_gem",
7
+ :password => "bunny_password",
8
+ :vhost => "bunny_testbed",
9
+ :socket_timeout => 0)
10
+ c.start
11
+ c
12
+ end
12
13
 
13
- after :all do
14
- connection.close if connection.open?
15
- end
14
+ after :all do
15
+ connection.close if connection.open?
16
+ end
16
17
 
17
18
 
18
- context "when publishing thousands of messages" do
19
- let(:n) { 10_000 }
20
- let(:m) { 10 }
19
+ context "when publishing thousands of messages" do
20
+ let(:n) { 2_000 }
21
+ let(:m) { 10 }
21
22
 
22
- it "successfully publishers them all" do
23
- ch = connection.create_channel
23
+ it "successfully publishers them all" do
24
+ ch = connection.create_channel
24
25
 
25
- q = ch.queue("", :exclusive => true)
26
- x = ch.default_exchange
26
+ q = ch.queue("", :exclusive => true)
27
+ x = ch.default_exchange
27
28
 
28
- body = "x" * 1024
29
- m.times do |i|
30
- n.times do
31
- x.publish(body, :routing_key => q.name)
29
+ body = "x" * 1024
30
+ m.times do |i|
31
+ n.times do
32
+ x.publish(body, :routing_key => q.name)
33
+ end
34
+ puts "Published #{i * n} 1K messages..."
32
35
  end
33
- puts "Published #{i * n} messages so far..."
34
- end
35
36
 
36
- q.purge
37
- ch.close
37
+ q.purge
38
+ ch.close
39
+ end
38
40
  end
39
41
  end
40
42
  end