beetle 0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. data/.gitignore +5 -0
  2. data/MIT-LICENSE +20 -0
  3. data/README.rdoc +82 -0
  4. data/Rakefile +114 -0
  5. data/TODO +7 -0
  6. data/beetle.gemspec +127 -0
  7. data/etc/redis-master.conf +189 -0
  8. data/etc/redis-slave.conf +189 -0
  9. data/examples/README.rdoc +14 -0
  10. data/examples/attempts.rb +66 -0
  11. data/examples/handler_class.rb +64 -0
  12. data/examples/handling_exceptions.rb +73 -0
  13. data/examples/multiple_exchanges.rb +48 -0
  14. data/examples/multiple_queues.rb +43 -0
  15. data/examples/redis_failover.rb +65 -0
  16. data/examples/redundant.rb +65 -0
  17. data/examples/rpc.rb +45 -0
  18. data/examples/simple.rb +39 -0
  19. data/lib/beetle.rb +57 -0
  20. data/lib/beetle/base.rb +78 -0
  21. data/lib/beetle/client.rb +252 -0
  22. data/lib/beetle/configuration.rb +31 -0
  23. data/lib/beetle/deduplication_store.rb +152 -0
  24. data/lib/beetle/handler.rb +95 -0
  25. data/lib/beetle/message.rb +336 -0
  26. data/lib/beetle/publisher.rb +187 -0
  27. data/lib/beetle/r_c.rb +40 -0
  28. data/lib/beetle/subscriber.rb +144 -0
  29. data/script/start_rabbit +29 -0
  30. data/snafu.rb +55 -0
  31. data/test/beetle.yml +81 -0
  32. data/test/beetle/base_test.rb +52 -0
  33. data/test/beetle/bla.rb +0 -0
  34. data/test/beetle/client_test.rb +305 -0
  35. data/test/beetle/configuration_test.rb +5 -0
  36. data/test/beetle/deduplication_store_test.rb +90 -0
  37. data/test/beetle/handler_test.rb +105 -0
  38. data/test/beetle/message_test.rb +744 -0
  39. data/test/beetle/publisher_test.rb +407 -0
  40. data/test/beetle/r_c_test.rb +9 -0
  41. data/test/beetle/subscriber_test.rb +263 -0
  42. data/test/beetle_test.rb +5 -0
  43. data/test/test_helper.rb +20 -0
  44. data/tmp/master/.gitignore +2 -0
  45. data/tmp/slave/.gitignore +3 -0
  46. metadata +192 -0
@@ -0,0 +1,81 @@
1
+ # list all standard exchanges used by the main xing app, along with their options for declaration
2
+ # used by producers and consumers
3
+ exchanges:
4
+ test:
5
+ type: "topic"
6
+ durable: true
7
+ deadletter:
8
+ type: "topic"
9
+ durable: true
10
+ redundant:
11
+ type: "topic"
12
+ durable: true
13
+
14
+ # list all standard queues along with their binding declaration
15
+ # this section is only used by consumers
16
+ queues:
17
+ test: # binding options
18
+ exchange: "test" # Bandersnatch default is the name of the queue
19
+ passive: false # amqp default is false
20
+ durable: true # amqp default is false
21
+ exclusive: false # amqp default is false
22
+ auto_delete: false # amqp default is false
23
+ nowait: true # amqp default is true
24
+ key: "#" # listen to every message
25
+ deadletter:
26
+ exchange: "deadletter"
27
+ durable: true
28
+ key: "#"
29
+ redundant:
30
+ exchange: "redundant"
31
+ durable: true
32
+ key: "#"
33
+ additional_queue:
34
+ exchange: "redundant"
35
+ durable: true
36
+ key: "#"
37
+
38
+ # list all messages we can publish
39
+ messages:
40
+ test:
41
+ queue: "test"
42
+ # Spefify the queue for listeners (default is message name)
43
+ key: "test"
44
+ # Specifies the routing key pattern for message subscription.
45
+ ttl: <%= 1.hour %>
46
+ # Specifies the time interval after which messages are silently dropped (seconds)
47
+ mandatory: true
48
+ # default is false
49
+ # Tells the server how to react if the message
50
+ # cannot be routed to a queue. If set to _true_, the server will return an unroutable message
51
+ # with a Return method. If this flag is zero, the server silently drops the message.
52
+ immediate: false
53
+ # default is false
54
+ # Tells the server how to react if the message
55
+ # cannot be routed to a queue consumer immediately. If set to _true_, the server will return an
56
+ # undeliverable message with a Return method. If set to _false_, the server will queue the message,
57
+ # but with no guarantee that it will ever be consumed.
58
+ persistent: true
59
+ # default is false
60
+ # Tells the server whether to persist the message
61
+ # If set to _true_, the message will be persisted to disk and not lost if the server restarts.
62
+ # If set to _false_, the message will not be persisted across server restart. Setting to _true_
63
+ # incurs a performance penalty as there is an extra cost associated with disk access.
64
+ deadletter:
65
+ key: "deadletter"
66
+ persistent: true
67
+ redundant:
68
+ key: "redundant"
69
+ persistent: true
70
+ redundant: true
71
+
72
+ development: &development
73
+ hostname: localhost:5672, localhost:5673
74
+ # hostname: localhost:5672
75
+ msg_id_store:
76
+ host: localhost
77
+ db: 4
78
+
79
+ test:
80
+ <<: *development
81
+ hostname: localhost:5672
@@ -0,0 +1,52 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
2
+
3
+
4
+ module Beetle
5
+ class BaseTest < Test::Unit::TestCase
6
+ test "initially we should have no exchanges" do
7
+ @bs = Base.new(Client.new)
8
+ assert_equal({}, @bs.instance_variable_get("@exchanges"))
9
+ end
10
+
11
+ test "initially we should have no queues" do
12
+ @bs = Base.new(Client.new)
13
+ assert_equal({}, @bs.instance_variable_get("@queues"))
14
+ end
15
+
16
+ test "the error method should raise a beetle error" do
17
+ @bs = Base.new(Client.new)
18
+ assert_raises(Error){ @bs.send(:error, "ha") }
19
+ end
20
+ end
21
+
22
+ class BaseServerManagementTest < Test::Unit::TestCase
23
+ def setup
24
+ @client = Client.new
25
+ @bs = Base.new(@client)
26
+ end
27
+
28
+ test "server should be initialized" do
29
+ assert_equal @bs.servers.first, @bs.server
30
+ end
31
+
32
+ test "current_host should return the hostname of the current server" do
33
+ @bs.server = "localhost:123"
34
+ assert_equal "localhost", @bs.send(:current_host)
35
+ end
36
+
37
+ test "current_port should return the port of the current server as an integer" do
38
+ @bs.server = "localhost:123"
39
+ assert_equal 123, @bs.send(:current_port)
40
+ end
41
+
42
+ test "current_port should return the default rabbit port if server string does not contain a port" do
43
+ @bs.server = "localhost"
44
+ assert_equal 5672, @bs.send(:current_port)
45
+ end
46
+
47
+ test "set_current_server shoud set the current server" do
48
+ @bs.send(:set_current_server, "xxx:123")
49
+ assert_equal "xxx:123", @bs.server
50
+ end
51
+ end
52
+ end
File without changes
@@ -0,0 +1,305 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
2
+
3
+
4
+ module Beetle
5
+ class ClientDefaultsTest < Test::Unit::TestCase
6
+ def setup
7
+ @client = Client.new
8
+ end
9
+
10
+ test "should have a default server" do
11
+ assert_equal ["localhost:5672"], @client.servers
12
+ end
13
+
14
+ test "should have no exchanges" do
15
+ assert @client.exchanges.empty?
16
+ end
17
+
18
+ test "should have no queues" do
19
+ assert @client.queues.empty?
20
+ end
21
+
22
+ test "should have no messages" do
23
+ assert @client.messages.empty?
24
+ end
25
+
26
+ test "should have no bindings" do
27
+ assert @client.bindings.empty?
28
+ end
29
+ end
30
+
31
+ class RegistrationTest < Test::Unit::TestCase
32
+ def setup
33
+ @client = Client.new
34
+ end
35
+
36
+ test "registering an exchange should store it in the configuration with symbolized option keys and force a topic queue and durability" do
37
+ opts = {"durable" => false, "type" => "fanout"}
38
+ @client.register_exchange("some_exchange", opts)
39
+ assert_equal({:durable => true, :type => :topic}, @client.exchanges["some_exchange"])
40
+ end
41
+
42
+ test "should convert exchange name to a string when registering an exchange" do
43
+ @client.register_exchange(:some_exchange)
44
+ assert(@client.exchanges.include?("some_exchange"))
45
+ end
46
+
47
+ test "registering an exchange should raise a configuration error if it is already configured" do
48
+ @client.register_exchange("some_exchange")
49
+ assert_raises(ConfigurationError){ @client.register_exchange("some_exchange") }
50
+ end
51
+
52
+ test "registering a queue should automatically register the corresponding exchange if it doesn't exist yet" do
53
+ @client.register_queue("some_queue", "durable" => false, "exchange" => "some_exchange")
54
+ assert @client.exchanges.include?("some_exchange")
55
+ end
56
+
57
+ test "registering a queue should store key and exchange in the bindings list" do
58
+ @client.register_queue(:some_queue, :key => "some_key", :exchange => "some_exchange")
59
+ assert_equal([{:key => "some_key", :exchange => "some_exchange"}], @client.bindings["some_queue"])
60
+ end
61
+
62
+ test "registering an additional binding for a queue should store key and exchange in the bindings list" do
63
+ @client.register_queue(:some_queue, :key => "some_key", :exchange => "some_exchange")
64
+ @client.register_binding(:some_queue, :key => "other_key", :exchange => "other_exchange")
65
+ bindings = @client.bindings["some_queue"]
66
+ expected_bindings = [{:key => "some_key", :exchange => "some_exchange"}, {:key => "other_key", :exchange => "other_exchange"}]
67
+ assert_equal expected_bindings, bindings
68
+ end
69
+
70
+ test "registering a queue should store it in the configuration with symbolized option keys and force durable=true and passive=false and set the amqp queue name" do
71
+ @client.register_queue("some_queue", "durable" => false, "exchange" => "some_exchange")
72
+ assert_equal({:durable => true, :passive => false, :auto_delete => false, :exclusive => false, :amqp_name => "some_queue"}, @client.queues["some_queue"])
73
+ end
74
+
75
+ test "registering a queue should add the queue to the list of queues of the queue's exchange" do
76
+ @client.register_queue("some_queue", "durable" => true, "exchange" => "some_exchange")
77
+ assert_equal ["some_queue"], @client.exchanges["some_exchange"][:queues]
78
+ end
79
+
80
+ test "registering two queues should add both queues to the list of queues of the queue's exchange" do
81
+ @client.register_queue("queue1", :exchange => "some_exchange")
82
+ @client.register_queue("queue2", :exchange => "some_exchange")
83
+ assert_equal ["queue1","queue2"], @client.exchanges["some_exchange"][:queues]
84
+ end
85
+
86
+ test "registering a queue should raise a configuration error if it is already configured" do
87
+ @client.register_queue("some_queue", "durable" => true, "exchange" => "some_exchange")
88
+ assert_raises(ConfigurationError){ @client.register_queue("some_queue") }
89
+ end
90
+
91
+ test "should convert queue name to a string when registering a queue" do
92
+ @client.register_queue(:some_queue)
93
+ assert(@client.queues.include?("some_queue"))
94
+ end
95
+
96
+ test "should convert exchange name to a string when registering a queue" do
97
+ @client.register_queue(:some_queue, :exchange => :murks)
98
+ assert_equal("murks", @client.bindings["some_queue"].first[:exchange])
99
+ end
100
+
101
+ test "registering a message should store it in the configuration with symbolized option keys" do
102
+ opts = {"persistent" => true, "queue" => "some_queue", "exchange" => "some_exchange"}
103
+ @client.register_queue("some_queue", "exchange" => "some_exchange")
104
+ @client.register_message("some_message", opts)
105
+ assert_equal({:persistent => true, :queue => "some_queue", :exchange => "some_exchange", :key => "some_message"}, @client.messages["some_message"])
106
+ end
107
+
108
+ test "registering a message should raise a configuration error if it is already configured" do
109
+ opts = {"persistent" => true, "queue" => "some_queue"}
110
+ @client.register_queue("some_queue", "exchange" => "some_exchange")
111
+ @client.register_message("some_message", opts)
112
+ assert_raises(ConfigurationError){ @client.register_message("some_message", opts) }
113
+ end
114
+
115
+ test "should convert message name to a string when registering a message" do
116
+ @client.register_message(:some_message)
117
+ assert(@client.messages.include?("some_message"))
118
+ end
119
+
120
+ test "should convert exchange name to a string when registering a message" do
121
+ @client.register_message(:some_message, :exchange => :murks)
122
+ assert_equal("murks", @client.messages["some_message"][:exchange])
123
+ end
124
+
125
+ test "configure should yield a configurator configured with the client and the given options" do
126
+ options = {:exchange => :foobar}
127
+ Client::Configurator.expects(:new).with(@client, options).returns(42)
128
+ @client.configure(options) {|config| assert_equal 42, config}
129
+ end
130
+
131
+ test "a configurator should forward all known registration methods to the client" do
132
+ options = {:foo => :bar}
133
+ config = Client::Configurator.new(@client, options)
134
+ @client.expects(:register_exchange).with(:a, options)
135
+ config.exchange(:a)
136
+
137
+ @client.expects(:register_queue).with(:q, options.merge(:exchange => :foo))
138
+ config.queue(:q, :exchange => :foo)
139
+
140
+ @client.expects(:register_binding).with(:b, options.merge(:key => :baz))
141
+ config.binding(:b, :key => :baz)
142
+
143
+ @client.expects(:register_message).with(:m, options.merge(:exchange => :foo))
144
+ config.message(:m, :exchange => :foo)
145
+
146
+ @client.expects(:register_handler).with(:h, options.merge(:queue => :q))
147
+ config.handler(:h, :queue => :q)
148
+
149
+ assert_raises(NoMethodError){ config.moo }
150
+ end
151
+ end
152
+
153
+ class ClientTest < Test::Unit::TestCase
154
+ test "instantiating a client should not instantiate the subscriber/publisher" do
155
+ Publisher.expects(:new).never
156
+ Subscriber.expects(:new).never
157
+ Client.new
158
+ end
159
+
160
+ test "should instantiate a subscriber when used for subscribing" do
161
+ Subscriber.expects(:new).returns(stub_everything("subscriber"))
162
+ client = Client.new
163
+ client.register_queue("superman")
164
+ client.register_message("superman")
165
+ client.register_handler("superman", {}, &lambda{})
166
+ end
167
+
168
+ test "should instantiate a subscriber when used for publishing" do
169
+ client = Client.new
170
+ client.register_message("foobar")
171
+ Publisher.expects(:new).returns(stub_everything("subscriber"))
172
+ client.publish("foobar", "payload")
173
+ end
174
+
175
+ test "should delegate publishing to the publisher instance" do
176
+ client = Client.new
177
+ client.register_message("deadletter")
178
+ args = ["deadletter", "x", {:a => 1}]
179
+ client.send(:publisher).expects(:publish).with(*args).returns(1)
180
+ assert_equal 1, client.publish(*args)
181
+ end
182
+
183
+ test "should convert message name to a string when publishing" do
184
+ client = Client.new
185
+ client.register_message("deadletter")
186
+ args = [:deadletter, "x", {:a => 1}]
187
+ client.send(:publisher).expects(:publish).with("deadletter", "x", :a => 1).returns(1)
188
+ assert_equal 1, client.publish(*args)
189
+ end
190
+
191
+ test "should convert message name to a string on rpc" do
192
+ client = Client.new
193
+ client.register_message("deadletter")
194
+ args = [:deadletter, "x", {:a => 1}]
195
+ client.send(:publisher).expects(:rpc).with("deadletter", "x", :a => 1).returns(1)
196
+ assert_equal 1, client.rpc(*args)
197
+ end
198
+
199
+ test "trying to publish an unknown message should raise an exception" do
200
+ assert_raises(UnknownMessage) { Client.new.publish("foobar") }
201
+ end
202
+
203
+ test "trying to RPC an unknown message should raise an exception" do
204
+ assert_raises(UnknownMessage) { Client.new.rpc("foobar") }
205
+ end
206
+
207
+ test "should delegate stop_publishing to the publisher instance" do
208
+ client = Client.new
209
+ client.send(:publisher).expects(:stop)
210
+ client.stop_publishing
211
+ end
212
+
213
+ test "should delegate queue purging to the publisher instance" do
214
+ client = Client.new
215
+ client.register_queue(:queue)
216
+ client.send(:publisher).expects(:purge).with("queue").returns("ha!")
217
+ assert_equal "ha!", client.purge("queue")
218
+ end
219
+
220
+ test "purging a queue should convert the queue name to a string" do
221
+ client = Client.new
222
+ client.register_queue(:queue)
223
+ client.send(:publisher).expects(:purge).with("queue").returns("ha!")
224
+ assert_equal "ha!", client.purge(:queue)
225
+ end
226
+
227
+ test "trying to purge an unknown queue should raise an exception" do
228
+ assert_raises(UnknownQueue) { Client.new.purge(:mumu) }
229
+ end
230
+
231
+ test "should delegate rpc calls to the publisher instance" do
232
+ client = Client.new
233
+ client.register_message("deadletter")
234
+ args = ["deadletter", "x", {:a => 1}]
235
+ client.send(:publisher).expects(:rpc).with(*args).returns("ha!")
236
+ assert_equal "ha!", client.rpc(*args)
237
+ end
238
+
239
+ test "should delegate listening to the subscriber instance" do
240
+ client = Client.new
241
+ client.register_queue(:a)
242
+ client.register_message(:a)
243
+ client.register_queue(:b)
244
+ client.register_message(:b)
245
+ client.send(:subscriber).expects(:listen).with(["a", "b"]).yields
246
+ x = 0
247
+ client.listen([:a, "b"]) { x = 5 }
248
+ assert_equal 5, x
249
+ end
250
+
251
+ test "trying to listen to an unknown message should raise an exception" do
252
+ assert_raises(UnknownMessage) { Client.new.listen([:a])}
253
+ end
254
+
255
+ test "should delegate stop_listening to the subscriber instance" do
256
+ client = Client.new
257
+ client.send(:subscriber).expects(:stop!)
258
+ client.stop_listening
259
+ end
260
+
261
+ test "should delegate handler registration to the subscriber instance" do
262
+ client = Client.new
263
+ client.register_queue("huhu")
264
+ client.send(:subscriber).expects(:register_handler)
265
+ client.register_handler("huhu")
266
+ end
267
+
268
+ test "should convert queue names to strings when registering a handler" do
269
+ client = Client.new
270
+ client.register_queue(:haha)
271
+ client.register_queue(:huhu)
272
+ client.send(:subscriber).expects(:register_handler).with(["huhu", "haha"], {}, nil)
273
+ client.register_handler([:huhu, :haha])
274
+ end
275
+
276
+ test "should use the configured logger" do
277
+ client = Client.new
278
+ Beetle.config.expects(:logger)
279
+ client.logger
280
+ end
281
+
282
+ test "load should expand the glob argument and evaluate each file in the client instance" do
283
+ client = Client.new
284
+ File.expects(:read).returns("1+1")
285
+ client.expects(:eval).with("1+1",anything,anything)
286
+ client.load("#{File.dirname(__FILE__)}/../../**/client_test.rb")
287
+ end
288
+
289
+ test "tracing should modify the amqp options for each queue and register a handler for each queue" do
290
+ client = Client.new
291
+ client.register_queue("test")
292
+ sub = client.send(:subscriber)
293
+ sub.expects(:register_handler).with(client.queues.keys, {}, nil).yields(stub_everything("message"))
294
+ sub.expects(:listen)
295
+ client.stubs(:puts)
296
+ client.trace
297
+ test_queue_opts = client.queues["test"]
298
+ expected_name = client.send :queue_name_for_tracing, "test"
299
+ assert_equal expected_name, test_queue_opts[:amqp_name]
300
+ assert test_queue_opts[:auto_delete]
301
+ assert !test_queue_opts[:durable]
302
+ end
303
+
304
+ end
305
+ end
@@ -0,0 +1,5 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
2
+
3
+
4
+ module Beetle
5
+ end
@@ -0,0 +1,90 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
2
+
3
+ module Beetle
4
+
5
+ class RedisAssumptionsTest < Test::Unit::TestCase
6
+ def setup
7
+ @r = DeduplicationStore.new.redis
8
+ @r.flushdb
9
+ end
10
+
11
+ test "trying to delete a non existent key doesn't throw an error" do
12
+ assert !@r.del("hahahaha")
13
+ assert !@r.exists("hahahaha")
14
+ end
15
+
16
+ test "msetnx returns 0 or 1" do
17
+ assert_equal 1, @r.msetnx("a" => 1, "b" => 2)
18
+ assert_equal "1", @r.get("a")
19
+ assert_equal "2", @r.get("b")
20
+ assert_equal 0, @r.msetnx("a" => 3, "b" => 4)
21
+ assert_equal "1", @r.get("a")
22
+ assert_equal "2", @r.get("b")
23
+ end
24
+ end
25
+
26
+ class RedisFailoverTest < Test::Unit::TestCase
27
+ def setup
28
+ @store = DeduplicationStore.new("localhost:1, localhost:2")
29
+ end
30
+
31
+ test "redis instances should be created for all servers" do
32
+ instances = @store.redis_instances
33
+ assert_equal ["localhost:1", "localhost:2" ], instances.map(&:server)
34
+ end
35
+
36
+ test "searching a redis master should find one if there is one" do
37
+ instances = @store.redis_instances
38
+ instances.first.expects(:info).returns(:role => "slave")
39
+ instances.second.expects(:info).returns(:role => "master")
40
+ assert_equal instances.second, @store.redis
41
+ end
42
+
43
+ test "searching a redis master should find one even if one cannot be accessed" do
44
+ instances = @store.redis_instances
45
+ instances.first.expects(:info).raises("murks")
46
+ instances.second.expects(:info).returns(:role => "master")
47
+ assert_equal instances.second, @store.redis
48
+ end
49
+
50
+ test "searching a redis master should raise an exception if there is none" do
51
+ instances = @store.redis_instances
52
+ instances.first.expects(:info).returns(:role => "slave")
53
+ instances.second.expects(:info).returns(:role => "slave")
54
+ assert_raises(NoRedisMaster) { @store.find_redis_master }
55
+ end
56
+
57
+ test "searching a redis master should raise an exception if there is more than one" do
58
+ instances = @store.redis_instances
59
+ instances.first.expects(:info).returns(:role => "master")
60
+ instances.second.expects(:info).returns(:role => "master")
61
+ assert_raises(TwoRedisMasters) { @store.find_redis_master }
62
+ end
63
+
64
+ test "a redis operation protected with a redis failover block should succeed if it can find a new master" do
65
+ instances = @store.redis_instances
66
+ s = sequence("redis accesses")
67
+ instances.first.expects(:info).returns(:role => "master").in_sequence(s)
68
+ instances.second.expects(:info).returns(:role => "slave").in_sequence(s)
69
+ assert_equal instances.first, @store.redis
70
+ instances.first.expects(:get).with("foo:x").raises("disconnected").in_sequence(s)
71
+ instances.first.expects(:info).raises("disconnected").in_sequence(s)
72
+ instances.second.expects(:info).returns(:role => "master").in_sequence(s)
73
+ instances.second.expects(:get).with("foo:x").returns("42").in_sequence(s)
74
+ assert_equal("42", @store.get("foo", "x"))
75
+ end
76
+
77
+ test "a redis operation protected with a redis failover block should fail if it cannot find a new master" do
78
+ instances = @store.redis_instances
79
+ instances.first.expects(:info).returns(:role => "master")
80
+ instances.second.expects(:info).returns(:role => "slave")
81
+ assert_equal instances.first, @store.redis
82
+ instances.first.stubs(:get).with("foo:x").raises("disconnected")
83
+ instances.first.stubs(:info).raises("disconnected")
84
+ instances.second.stubs(:info).returns(:role => "slave")
85
+ @store.expects(:sleep).times(119)
86
+ assert_raises(NoRedisMaster) { @store.get("foo", "x") }
87
+ end
88
+ end
89
+
90
+ end