beetle 0.1 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. data/README.rdoc +18 -8
  2. data/beetle.gemspec +37 -121
  3. data/bin/beetle +9 -0
  4. data/examples/README.rdoc +0 -2
  5. data/examples/rpc.rb +3 -2
  6. data/ext/mkrf_conf.rb +19 -0
  7. data/lib/beetle.rb +2 -2
  8. data/lib/beetle/base.rb +1 -8
  9. data/lib/beetle/client.rb +16 -14
  10. data/lib/beetle/commands.rb +30 -0
  11. data/lib/beetle/commands/configuration_client.rb +73 -0
  12. data/lib/beetle/commands/configuration_server.rb +85 -0
  13. data/lib/beetle/configuration.rb +70 -7
  14. data/lib/beetle/deduplication_store.rb +50 -38
  15. data/lib/beetle/handler.rb +2 -5
  16. data/lib/beetle/logging.rb +7 -0
  17. data/lib/beetle/message.rb +11 -13
  18. data/lib/beetle/publisher.rb +12 -4
  19. data/lib/beetle/r_c.rb +2 -1
  20. data/lib/beetle/redis_configuration_client.rb +136 -0
  21. data/lib/beetle/redis_configuration_server.rb +301 -0
  22. data/lib/beetle/redis_ext.rb +79 -0
  23. data/lib/beetle/redis_master_file.rb +35 -0
  24. data/lib/beetle/redis_server_info.rb +65 -0
  25. data/lib/beetle/subscriber.rb +4 -1
  26. data/test/beetle/configuration_test.rb +14 -2
  27. data/test/beetle/deduplication_store_test.rb +61 -43
  28. data/test/beetle/message_test.rb +28 -4
  29. data/test/beetle/publisher_test.rb +17 -3
  30. data/test/beetle/redis_configuration_client_test.rb +97 -0
  31. data/test/beetle/redis_configuration_server_test.rb +278 -0
  32. data/test/beetle/redis_ext_test.rb +71 -0
  33. data/test/beetle/redis_master_file_test.rb +39 -0
  34. data/test/test_helper.rb +13 -1
  35. metadata +162 -69
  36. data/.gitignore +0 -5
  37. data/MIT-LICENSE +0 -20
  38. data/Rakefile +0 -114
  39. data/TODO +0 -7
  40. data/etc/redis-master.conf +0 -189
  41. data/etc/redis-slave.conf +0 -189
  42. data/examples/redis_failover.rb +0 -65
  43. data/script/start_rabbit +0 -29
  44. data/snafu.rb +0 -55
  45. data/test/beetle.yml +0 -81
  46. data/test/beetle/bla.rb +0 -0
  47. data/tmp/master/.gitignore +0 -2
  48. data/tmp/slave/.gitignore +0 -3
@@ -0,0 +1,79 @@
1
+ # Redis convenience and compatibility layer
2
+ class Redis #:nodoc:
3
+ def self.from_server_string(server_string, options = {})
4
+ host, port = server_string.split(':')
5
+ options = {:host => host, :port => port}.update(options)
6
+ new(options)
7
+ end
8
+
9
+ # Redis 2 removed some useful methods. add them back.
10
+ def host; @client.host; end
11
+ def port; @client.port; end
12
+ def server; "#{host}:#{port}"; end
13
+
14
+ def master!
15
+ slaveof("no", "one")
16
+ end
17
+
18
+ def slave_of!(host, port)
19
+ slaveof(host, port)
20
+ end
21
+
22
+ # Redis 2 tries to establish a connection on inspect. this is evil!
23
+ def inspect
24
+ super
25
+ end
26
+
27
+ def info_with_rescue
28
+ info
29
+ rescue Exception
30
+ {}
31
+ end
32
+
33
+ def available?
34
+ info_with_rescue != {}
35
+ end
36
+
37
+ def role
38
+ info_with_rescue["role"] || "unknown"
39
+ end
40
+
41
+ def master?
42
+ role == "master"
43
+ end
44
+
45
+ def slave?
46
+ role == "slave"
47
+ end
48
+
49
+ def slave_of?(host, port)
50
+ info = info_with_rescue
51
+ info["role"] == "slave" && info["master_host"] == host && info["master_port"] == port.to_s
52
+ end
53
+ end
54
+
55
+ class Redis::Client #:nodoc:
56
+ protected
57
+ def connect_to(host, port)
58
+ if @timeout != 0 and Redis::Timer
59
+ begin
60
+ Redis::Timer.timeout(@timeout){ @sock = TCPSocket.new(host, port) }
61
+ rescue Timeout::Error
62
+ @sock = nil
63
+ raise Timeout::Error, "Timeout connecting to the server"
64
+ end
65
+ else
66
+ @sock = TCPSocket.new(host, port)
67
+ end
68
+
69
+ @sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1
70
+
71
+ # If the timeout is set we set the low level socket options in order
72
+ # to make sure a blocking read will return after the specified number
73
+ # of seconds. This hack is from memcached ruby client.
74
+ self.timeout = @timeout
75
+
76
+ rescue Errno::ECONNREFUSED
77
+ raise Errno::ECONNREFUSED, "Unable to connect to Redis on #{host}:#{port}"
78
+ end
79
+ end
@@ -0,0 +1,35 @@
1
+ module Beetle
2
+ module RedisMasterFile #:nodoc:
3
+ private
4
+ def master_file_exists?
5
+ File.exist?(master_file)
6
+ end
7
+
8
+ def redis_master_from_master_file
9
+ redis.find(read_redis_master_file)
10
+ end
11
+
12
+ def clear_redis_master_file
13
+ write_redis_master_file("")
14
+ end
15
+
16
+ def read_redis_master_file
17
+ File.read(master_file).chomp
18
+ end
19
+
20
+ def write_redis_master_file(redis_server_string)
21
+ logger.warn "Writing '#{redis_server_string}' to redis master file '#{master_file}'"
22
+ File.open(master_file, "w"){|f| f.puts redis_server_string }
23
+ end
24
+
25
+ def master_file
26
+ config.redis_server
27
+ end
28
+
29
+ def verify_redis_master_file_string
30
+ if master_file =~ /^[0-9a-z.]+:[0-9]+$/
31
+ raise ConfigurationError.new("To use the redis failover, redis_server config option must point to a file")
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,65 @@
1
+ module Beetle
2
+ # class used by the RedisConfigurationServer to hold information about the current state of the configured redis servers
3
+ class RedisServerInfo
4
+ include Logging
5
+
6
+ def initialize(config, options) #:nodoc:
7
+ @config = config
8
+ @options = options
9
+ reset
10
+ end
11
+
12
+ # all configured redis servers
13
+ def instances
14
+ @instances ||= @config.redis_servers.split(/ *, */).map{|s| Redis.from_server_string(s, @options)}
15
+ end
16
+
17
+ # fetches the server from the insatnces whith the given <tt>server</tt> string
18
+ def find(server)
19
+ instances.find{|r| r.server == server}
20
+ end
21
+
22
+ # refresh connectivity/role information
23
+ def refresh
24
+ logger.debug "Updating redis server info"
25
+ reset
26
+ instances.each {|r| @server_info[r.role] << r}
27
+ end
28
+
29
+ # subset of instances which are masters
30
+ def masters
31
+ @server_info["master"]
32
+ end
33
+
34
+ # subset of instances which are slaves
35
+ def slaves
36
+ @server_info["slave"]
37
+ end
38
+
39
+ # subset of instances which are not reachable
40
+ def unknowns
41
+ @server_info["unknown"]
42
+ end
43
+
44
+ # subset of instances which are set up as slaves of the given <tt>master</tt>
45
+ def slaves_of(master)
46
+ slaves.select{|r| r.slave_of?(master.host, master.port)}
47
+ end
48
+
49
+ # determine a master if we have one master and all other servers are slaves
50
+ def auto_detect_master
51
+ return nil unless master_and_slaves_reachable?
52
+ masters.first
53
+ end
54
+
55
+ private
56
+
57
+ def master_and_slaves_reachable?
58
+ masters.size == 1 && slaves.size == instances.size - 1
59
+ end
60
+
61
+ def reset
62
+ @server_info = Hash.new {|h,k| h[k]= []}
63
+ end
64
+ end
65
+ end
@@ -10,7 +10,7 @@ module Beetle
10
10
  @mqs = {}
11
11
  end
12
12
 
13
- # the client calls this method to subcsribe to all queues on all servers which have
13
+ # the client calls this method to subscribe to all queues on all servers which have
14
14
  # handlers registered for the given list of messages. this method does the following
15
15
  # things:
16
16
  #
@@ -105,6 +105,9 @@ module Beetle
105
105
  sleep 1
106
106
  mq(server).recover
107
107
  elsif reply_to = header.properties[:reply_to]
108
+ # require 'ruby-debug'
109
+ # Debugger.start
110
+ # debugger
108
111
  status = result == Beetle::RC::OK ? "OK" : "FAILED"
109
112
  exchange = MQ::Exchange.new(mq(server), :direct, "", :key => reply_to)
110
113
  exchange.publish(m.handler_result.to_s, :headers => {:status => status})
@@ -1,5 +1,17 @@
1
1
  require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
2
-
2
+ require 'tempfile'
3
3
 
4
4
  module Beetle
5
- end
5
+ class ConfigurationTest < Test::Unit::TestCase
6
+ test "should load it's settings from a config file if that file exists" do
7
+ config = Configuration.new
8
+ old_value = config.gc_threshold
9
+ new_value = old_value + 1
10
+ config_file_content = "gc_threshold: #{new_value}\n"
11
+ IO.expects(:read).returns(config_file_content)
12
+
13
+ config.config_file = "some/path/to/a/file"
14
+ assert_equal new_value, config.gc_threshold
15
+ end
16
+ end
17
+ end
@@ -14,75 +14,93 @@ module Beetle
14
14
  end
15
15
 
16
16
  test "msetnx returns 0 or 1" do
17
- assert_equal 1, @r.msetnx("a" => 1, "b" => 2)
17
+ assert_equal 1, @r.msetnx("a", 1, "b", 2)
18
18
  assert_equal "1", @r.get("a")
19
19
  assert_equal "2", @r.get("b")
20
- assert_equal 0, @r.msetnx("a" => 3, "b" => 4)
20
+ assert_equal 0, @r.msetnx("a", 3, "b", 4)
21
21
  assert_equal "1", @r.get("a")
22
22
  assert_equal "2", @r.get("b")
23
23
  end
24
24
  end
25
25
 
26
- class RedisFailoverTest < Test::Unit::TestCase
26
+ class RedisServerStringTest < Test::Unit::TestCase
27
+ def setup
28
+ @original_redis_server = Beetle.config.redis_server
29
+ @store = DeduplicationStore.new
30
+ @server_string = "my_test_host_from_file:9999"
31
+ Beetle.config.redis_server = @server_string
32
+ end
33
+
34
+ def teardown
35
+ Beetle.config.redis_server = @original_redis_server
36
+ end
37
+
38
+ test "redis should match the redis server string" do
39
+ assert_equal @server_string, @store.redis.server
40
+ end
41
+ end
42
+
43
+ class RedisServerFileTest < Test::Unit::TestCase
27
44
  def setup
28
- @store = DeduplicationStore.new("localhost:1, localhost:2")
45
+ @original_redis_server = Beetle.config.redis_server
46
+ @store = DeduplicationStore.new
47
+ @server_string = "my_test_host_from_file:6379"
48
+ Beetle.config.redis_server = redis_test_master_file(@server_string)
49
+ end
50
+
51
+ def teardown
52
+ Beetle.config.redis_server = @original_redis_server
53
+ end
54
+
55
+ test "redis should match the redis master file" do
56
+ assert_equal @server_string, @store.redis.server
29
57
  end
30
58
 
31
- test "redis instances should be created for all servers" do
32
- instances = @store.redis_instances
33
- assert_equal ["localhost:1", "localhost:2" ], instances.map(&:server)
59
+ test "redis should be nil if the redis master file is blank" do
60
+ redis_test_master_file("")
61
+ assert_nil @store.redis
34
62
  end
35
63
 
36
- test "searching a redis master should find one if there is one" do
37
- instances = @store.redis_instances
38
- instances.first.expects(:info).returns(:role => "slave")
39
- instances.second.expects(:info).returns(:role => "master")
40
- assert_equal instances.second, @store.redis
64
+ test "should keep using the current redis if the redis master file hasn't changed since the last request" do
65
+ @store.expects(:read_master_file).once.returns("localhost:1")
66
+ 2.times { @store.redis }
41
67
  end
42
68
 
43
- test "searching a redis master should find one even if one cannot be accessed" do
44
- instances = @store.redis_instances
45
- instances.first.expects(:info).raises("murks")
46
- instances.second.expects(:info).returns(:role => "master")
47
- assert_equal instances.second, @store.redis
69
+ test "should return nil if the master file doesn't exist" do
70
+ Beetle.config.redis_server = "/tmp/__i_don_not_exist__.txt"
71
+ assert_equal nil, @store.redis_master_from_master_file
48
72
  end
49
73
 
50
- test "searching a redis master should raise an exception if there is none" do
51
- instances = @store.redis_instances
52
- instances.first.expects(:info).returns(:role => "slave")
53
- instances.second.expects(:info).returns(:role => "slave")
54
- assert_raises(NoRedisMaster) { @store.find_redis_master }
74
+ private
75
+ def redis_test_master_file(server_string)
76
+ path = File.expand_path("../../../tmp/redis-master-for-unit-tests", __FILE__)
77
+ File.open(path, "w"){|f| f.puts server_string}
78
+ path
55
79
  end
80
+ end
56
81
 
57
- test "searching a redis master should raise an exception if there is more than one" do
58
- instances = @store.redis_instances
59
- instances.first.expects(:info).returns(:role => "master")
60
- instances.second.expects(:info).returns(:role => "master")
61
- assert_raises(TwoRedisMasters) { @store.find_redis_master }
82
+ class RedisFailoverTest < Test::Unit::TestCase
83
+ def setup
84
+ @store = DeduplicationStore.new
85
+ Beetle.config.expects(:redis_failover_timeout).returns(1)
62
86
  end
63
87
 
64
88
  test "a redis operation protected with a redis failover block should succeed if it can find a new master" do
65
- instances = @store.redis_instances
89
+ redis1 = stub("redis 1")
90
+ redis2 = stub("redis 2")
66
91
  s = sequence("redis accesses")
67
- instances.first.expects(:info).returns(:role => "master").in_sequence(s)
68
- instances.second.expects(:info).returns(:role => "slave").in_sequence(s)
69
- assert_equal instances.first, @store.redis
70
- instances.first.expects(:get).with("foo:x").raises("disconnected").in_sequence(s)
71
- instances.first.expects(:info).raises("disconnected").in_sequence(s)
72
- instances.second.expects(:info).returns(:role => "master").in_sequence(s)
73
- instances.second.expects(:get).with("foo:x").returns("42").in_sequence(s)
92
+ @store.expects(:redis).returns(redis1).in_sequence(s)
93
+ redis1.expects(:get).with("foo:x").raises("disconnected").in_sequence(s)
94
+ @store.expects(:redis).returns(redis2).in_sequence(s)
95
+ redis2.expects(:get).with("foo:x").returns("42").in_sequence(s)
74
96
  assert_equal("42", @store.get("foo", "x"))
75
97
  end
76
98
 
77
99
  test "a redis operation protected with a redis failover block should fail if it cannot find a new master" do
78
- instances = @store.redis_instances
79
- instances.first.expects(:info).returns(:role => "master")
80
- instances.second.expects(:info).returns(:role => "slave")
81
- assert_equal instances.first, @store.redis
82
- instances.first.stubs(:get).with("foo:x").raises("disconnected")
83
- instances.first.stubs(:info).raises("disconnected")
84
- instances.second.stubs(:info).returns(:role => "slave")
85
- @store.expects(:sleep).times(119)
100
+ redis1 = stub()
101
+ @store.stubs(:redis).returns(redis1)
102
+ redis1.stubs(:get).with("foo:x").raises("disconnected")
103
+ @store.stubs(:sleep)
86
104
  assert_raises(NoRedisMaster) { @store.get("foo", "x") }
87
105
  end
88
106
  end
@@ -4,6 +4,11 @@ require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
4
4
  module Beetle
5
5
 
6
6
  class EncodingTest < Test::Unit::TestCase
7
+ test "an exception during decoding should be stored in the exception attribute" do
8
+ header = stub_everything("raising header")
9
+ m = Message.new("queue", header, 'foo')
10
+ assert_instance_of NoMethodError, m.exception
11
+ end
7
12
 
8
13
  test "a message should encode/decode the message format version correctly" do
9
14
  header = header_with_params({})
@@ -371,6 +376,17 @@ module Beetle
371
376
  @store.flushdb
372
377
  end
373
378
 
379
+ test "a message with an exception set should not be processed at all, but it should be acked" do
380
+ header = {}
381
+ message = Message.new("somequeue", header, 'foo')
382
+ assert message.exception
383
+
384
+ proc = mock("proc")
385
+ proc.expects(:call).never
386
+ message.expects(:ack!)
387
+ assert_equal RC::DecodingError, message.__send__(:process_internal, proc)
388
+ end
389
+
374
390
  test "a completed existing message should be just acked and not run the handler" do
375
391
  header = header_with_params({})
376
392
  message = Message.new("somequeue", header, 'foo', :attempts => 2, :store => @store)
@@ -379,7 +395,6 @@ module Beetle
379
395
  assert message.completed?
380
396
 
381
397
  proc = mock("proc")
382
- s = sequence("s")
383
398
  header.expects(:ack)
384
399
  proc.expects(:call).never
385
400
  assert_equal RC::OK, message.__send__(:process_internal, proc)
@@ -394,7 +409,6 @@ module Beetle
394
409
  assert message.delayed?
395
410
 
396
411
  proc = mock("proc")
397
- s = sequence("s")
398
412
  header.expects(:ack).never
399
413
  proc.expects(:call).never
400
414
  assert_equal RC::Delayed, message.__send__(:process_internal, proc)
@@ -412,7 +426,6 @@ module Beetle
412
426
  assert !message.timed_out?
413
427
 
414
428
  proc = mock("proc")
415
- s = sequence("s")
416
429
  header.expects(:ack).never
417
430
  proc.expects(:call).never
418
431
  assert_equal RC::HandlerNotYetTimedOut, message.__send__(:process_internal, proc)
@@ -577,7 +590,7 @@ module Beetle
577
590
  @store.flushdb
578
591
  end
579
592
 
580
- test "a handler running longer than the specified timeout should be aborted" do
593
+ test "a handler running longer than the specified timeout should be aborted (when given a float timeout number)" do
581
594
  header = header_with_params({})
582
595
  header.expects(:ack)
583
596
  message = Message.new("somequeue", header, 'foo', :timeout => 0.1, :attempts => 2, :store => @store)
@@ -586,6 +599,17 @@ module Beetle
586
599
  result = message.process(handler)
587
600
  assert_equal RC::ExceptionsLimitReached, result
588
601
  end
602
+
603
+ test "a handler running longer than the specified timeout should be aborted (when using active_support seconds)" do
604
+ header = header_with_params({})
605
+ header.expects(:ack)
606
+ message = Message.new("somequeue", header, 'foo', :timeout => 1.seconds, :attempts => 2, :store => @store)
607
+ action = lambda{|*args| while true; end}
608
+ handler = Handler.create(action)
609
+ result = message.process(handler)
610
+ assert_equal RC::ExceptionsLimitReached, result
611
+ end
612
+
589
613
  end
590
614
 
591
615
  class SettingsTest < Test::Unit::TestCase
@@ -284,7 +284,7 @@ module Beetle
284
284
  assert_kind_of Time, dead["localhost:2222"]
285
285
  end
286
286
 
287
- test "recycle_dead_servers should move servers from the dead server hash to the servers list only if the have been markd dead for longer than 10 seconds" do
287
+ test "recycle_dead_servers should move servers from the dead server hash to the servers list only if they have been markd dead for longer than 10 seconds" do
288
288
  @pub.servers = ["a:1", "b:2"]
289
289
  @pub.send(:set_current_server, "a:1")
290
290
  @pub.send(:mark_server_dead)
@@ -299,6 +299,19 @@ module Beetle
299
299
  assert_equal({}, dead)
300
300
  end
301
301
 
302
+ test "recycle_dead_servers should recycle the server which has been dead for the longest time if all servers are dead " do
303
+ @pub.servers = ["a:1", "b:2"]
304
+ @pub.send(:set_current_server, "a:1")
305
+ @pub.send(:mark_server_dead)
306
+ @pub.send(:mark_server_dead)
307
+ assert_equal [], @pub.servers
308
+ dead = @pub.instance_variable_get("@dead_servers")
309
+ assert_equal ["a:1", "b:2"], dead.keys.sort
310
+ @pub.send(:recycle_dead_servers)
311
+ assert_equal ["b:2"], dead.keys
312
+ assert_equal ["a:1"], @pub.servers
313
+ end
314
+
302
315
  test "select_next_server should cycle through the list of all servers" do
303
316
  @pub.servers = ["a:1", "b:2"]
304
317
  @pub.send(:set_current_server, "a:1")
@@ -308,12 +321,13 @@ module Beetle
308
321
  assert_equal "a:1", @pub.server
309
322
  end
310
323
 
311
- test "select_next_server should return 0 if there are no servers to publish to" do
324
+ test "select_next_server should log an error if there are no servers to publish to" do
312
325
  @pub.servers = []
313
326
  logger = mock('logger')
314
327
  logger.expects(:error).returns(true)
315
328
  @pub.expects(:logger).returns(logger)
316
- assert_equal 0, @pub.send(:select_next_server)
329
+ @pub.expects(:set_current_server).never
330
+ @pub.send(:select_next_server)
317
331
  end
318
332
 
319
333
  test "stop should shut down all bunnies" do